columns = new HashSet<>();
+ columns.addAll(enabled.getColumnPaths());
+ columns.addAll(vectorSize.getColumnPaths());
+ for (ColumnPath column : columns) {
+ builder.withValue(column, new AlpConfig(enabled.getValue(column), vectorSize.getValue(column)));
+ }
+ return builder.build();
+ }
+
/**
* Set the Parquet format dictionary page size.
*
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConfig.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConfig.java
new file mode 100644
index 0000000000..236b45911c
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConfig.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import java.util.Objects;
+
+/**
+ * Immutable per-column ALP encoding configuration: whether ALP is enabled and the vector size
+ * (number of values per encoded vector) to use. Bundled together so a column carries a single
+ * cohesive ALP setting rather than several independent properties.
+ */
+public class AlpConfig {
+
+ private final boolean enabled;
+ private final int vectorSize;
+
+ public AlpConfig(boolean enabled, int vectorSize) {
+ this.enabled = enabled;
+ this.vectorSize = vectorSize;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public int getVectorSize() {
+ return vectorSize;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ AlpConfig that = (AlpConfig) o;
+ return enabled == that.enabled && vectorSize == that.vectorSize;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(enabled, vectorSize);
+ }
+
+ @Override
+ public String toString() {
+ return "AlpConfig{enabled=" + enabled + ", vectorSize=" + vectorSize + '}';
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConstants.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConstants.java
new file mode 100644
index 0000000000..e5dd652428
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConstants.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import org.apache.parquet.Preconditions;
+
+/**
+ * Constants for the ALP (Adaptive Lossless floating-Point) encoding.
+ *
+ * ALP encoding converts floating-point values to integers using decimal scaling,
+ * then applies Frame of Reference encoding and bit-packing.
+ * Values that cannot be losslessly converted are stored as exceptions.
+ *
+ *
Based on the paper: "ALP: Adaptive Lossless floating-Point Compression" (SIGMOD 2024)
+ *
+ * @see ALP Paper
+ */
+public final class AlpConstants {
+
+ private AlpConstants() {
+ // Utility class
+ }
+
+ // Page header fields
+ public static final int ALP_COMPRESSION_MODE = 0;
+ public static final int ALP_INTEGER_ENCODING_FOR = 0;
+ public static final int ALP_HEADER_SIZE = 7;
+
+ public static final int DEFAULT_VECTOR_SIZE = 1024;
+ public static final int DEFAULT_VECTOR_SIZE_LOG = 10;
+
+ // BytePacker packs/unpacks 8 values at a time (pack8Values/unpack8Values).
+ static final int PACK_GROUP_SIZE = 8;
+
+ // Capped at 15 (vectorSize=32768) because num_exceptions is uint16,
+ // so vectorSize must not exceed 65535 to avoid overflow when all values are exceptions.
+ static final int MAX_LOG_VECTOR_SIZE = 15;
+ static final int MIN_LOG_VECTOR_SIZE = 3;
+
+ static final int FLOAT_MAX_EXPONENT = 10;
+ static final int DOUBLE_MAX_EXPONENT = 18;
+
+ // Sampler constants matching C++ AlpConstants.
+ // Sample SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP vectors evenly distributed across a rowgroup
+ // of SAMPLER_ROWGROUP_SIZE values, then lock in top MAX_PRESET_COMBINATIONS combos.
+ static final int SAMPLER_ROWGROUP_SIZE = 122_880;
+ static final int SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP = 8;
+ static final int MAX_PRESET_COMBINATIONS = 5;
+
+ // Magic numbers for the fast-rounding trick (see ALP paper, Section 3.2)
+ static final float MAGIC_FLOAT = 12_582_912.0f; // 2^22 + 2^23
+ static final double MAGIC_DOUBLE = 6_755_399_441_055_744.0; // 2^51 + 2^52
+
+ // Per-vector metadata sizes in bytes
+ public static final int ALP_INFO_SIZE = 4; // exponent(1) + factor(1) + num_exceptions(2)
+ public static final int FLOAT_FOR_INFO_SIZE = 5; // frame_of_reference(4) + bit_width(1)
+ public static final int DOUBLE_FOR_INFO_SIZE = 9; // frame_of_reference(8) + bit_width(1)
+
+ // POWERS_OF_TEN: positive powers used for scaling up during encode/decode.
+ // Encode: fastRound(value * POW10[e] * POW10_NEGATIVE[f])
+ // Decode: encoded * POW10[f] * POW10_NEGATIVE[e]
+ static final float[] FLOAT_POW10 = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f};
+
+ static final double[] DOUBLE_POW10 = {
+ 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18
+ };
+
+ // NEGATIVE_POWERS_OF_TEN: reciprocals used for scaling down (multiply-by-reciprocal).
+ // Multiplying by a precomputed reciprocal instead of dividing keeps the rounding bit-for-bit
+ // identical to the ALP reference implementation, so encoded values interop across implementations.
+ static final float[] FLOAT_POW10_NEGATIVE = {
+ 1e0f, 1e-1f, 1e-2f, 1e-3f, 1e-4f, 1e-5f, 1e-6f, 1e-7f, 1e-8f, 1e-9f, 1e-10f
+ };
+
+ static final double[] DOUBLE_POW10_NEGATIVE = {
+ 1e0, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15, 1e-16,
+ 1e-17, 1e-18
+ };
+
+ static final int FLOAT_NEGATIVE_ZERO_BITS = 0x80000000;
+ static final long DOUBLE_NEGATIVE_ZERO_BITS = 0x8000000000000000L;
+
+ /** Validates vector size: must be a power of 2 in [2^MIN_LOG .. 2^MAX_LOG]. */
+ public static int validateVectorSize(int vectorSize) {
+ Preconditions.checkArgument(
+ vectorSize > 0 && (vectorSize & (vectorSize - 1)) == 0,
+ "Vector size must be a power of 2, got: %s",
+ vectorSize);
+ int logSize = Integer.numberOfTrailingZeros(vectorSize);
+ Preconditions.checkArgument(
+ logSize >= MIN_LOG_VECTOR_SIZE && logSize <= MAX_LOG_VECTOR_SIZE,
+ "Vector size log2 must be between %s and %s, got: %s (vectorSize=%s)",
+ MIN_LOG_VECTOR_SIZE,
+ MAX_LOG_VECTOR_SIZE,
+ logSize,
+ vectorSize);
+ return vectorSize;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpEncoderDecoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpEncoderDecoder.java
new file mode 100644
index 0000000000..8b6e0b0f7d
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpEncoderDecoder.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.*;
+
+/**
+ * Core ALP (Adaptive Lossless floating-Point) encoding and decoding logic.
+ *
+ *
ALP works by converting floating-point values to integers using decimal scaling,
+ * then applying Frame of Reference encoding and bit-packing.
+ * Values that cannot be losslessly converted are stored as exceptions.
+ *
+ *
Encoding formula: encoded = fastRound(value * POW10[e] * POW10_NEGATIVE[f])
+ *
Decoding formula: value = encoded * POW10[f] * POW10_NEGATIVE[e]
+ *
+ *
The order of operations is critical for IEEE 754 correctness. Both formulas must
+ * be evaluated as single expressions — storing the intermediate multiplication result
+ * in a variable before the second multiply changes IEEE 754 rounding and produces extra
+ * exceptions. Likewise, scaling uses multiply-by-reciprocal (via POW10_NEGATIVE) rather than
+ * division: this reproduces the exact IEEE 754 rounding of the ALP reference algorithm, so the
+ * encoded integers — and therefore which values become exceptions and the resulting bytes — are
+ * identical across implementations. It is about cross-implementation determinism, not any one
+ * language.
+ *
+ *
Exception conditions:
+ *
+ * - NaN values
+ * - Infinity values
+ * - Negative zero (-0.0)
+ * - Out of integer range
+ * - Round-trip failure (decode(encode(v)) != v)
+ *
+ */
+final class AlpEncoderDecoder {
+
+ private static final double ENCODING_UPPER_LIMIT = 9223372036854774784.0;
+ private static final double ENCODING_LOWER_LIMIT = -9223372036854774784.0;
+ private static final float FLOAT_ENCODING_UPPER_LIMIT = 2147483520.0f;
+ private static final float FLOAT_ENCODING_LOWER_LIMIT = -2147483520.0f;
+
+ private AlpEncoderDecoder() {
+ // Utility class
+ }
+
+ /** NaN, Inf, and -0.0 can never be encoded regardless of exponent/factor. */
+ static boolean isIntrinsicFloatException(float value) {
+ if (Float.isNaN(value)) {
+ return true;
+ }
+ if (Float.isInfinite(value)) {
+ return true;
+ }
+ return Float.floatToRawIntBits(value) == FLOAT_NEGATIVE_ZERO_BITS;
+ }
+
+ /** Full exception check for a given (exponent, factor): intrinsic cases plus round-trip failure. */
+ static boolean isFloatException(float value, int exponent, int factor) {
+ if (isIntrinsicFloatException(value)) {
+ return true;
+ }
+ // Check before rounding: overflow or non-finite after scaling
+ float scaled = value * FLOAT_POW10[exponent] * FLOAT_POW10_NEGATIVE[factor];
+ if (!Float.isFinite(scaled) || scaled > FLOAT_ENCODING_UPPER_LIMIT || scaled < FLOAT_ENCODING_LOWER_LIMIT) {
+ return true;
+ }
+ int encoded = encodeFloat(value, exponent, factor);
+ float decoded = decodeFloat(encoded, exponent, factor);
+ return Float.floatToRawIntBits(value) != Float.floatToRawIntBits(decoded);
+ }
+
+ /** Round float to nearest integer using magic-number trick with sign branching. */
+ static int fastRoundFloat(float value) {
+ if (value >= 0) {
+ return (int) ((value + MAGIC_FLOAT) - MAGIC_FLOAT);
+ } else {
+ return (int) ((value - MAGIC_FLOAT) + MAGIC_FLOAT);
+ }
+ }
+
+ static int encodeFloat(float value, int exponent, int factor) {
+ return fastRoundFloat(value * FLOAT_POW10[exponent] * FLOAT_POW10_NEGATIVE[factor]);
+ }
+
+ static float decodeFloat(int encoded, int exponent, int factor) {
+ return encoded * FLOAT_POW10[factor] * FLOAT_POW10_NEGATIVE[exponent];
+ }
+
+ /** NaN, Inf, and -0.0 can never be encoded regardless of exponent/factor. */
+ static boolean isIntrinsicDoubleException(double value) {
+ if (Double.isNaN(value)) {
+ return true;
+ }
+ if (Double.isInfinite(value)) {
+ return true;
+ }
+ return Double.doubleToRawLongBits(value) == DOUBLE_NEGATIVE_ZERO_BITS;
+ }
+
+ /** Full exception check for a given (exponent, factor): intrinsic cases plus round-trip failure. */
+ static boolean isDoubleException(double value, int exponent, int factor) {
+ if (isIntrinsicDoubleException(value)) {
+ return true;
+ }
+ // Check before rounding: overflow or non-finite after scaling
+ double scaled = value * DOUBLE_POW10[exponent] * DOUBLE_POW10_NEGATIVE[factor];
+ if (!Double.isFinite(scaled) || scaled > ENCODING_UPPER_LIMIT || scaled < ENCODING_LOWER_LIMIT) {
+ return true;
+ }
+ long encoded = encodeDouble(value, exponent, factor);
+ double decoded = decodeDouble(encoded, exponent, factor);
+ return Double.doubleToRawLongBits(value) != Double.doubleToRawLongBits(decoded);
+ }
+
+ /** Round double to nearest integer using magic-number trick with sign branching. */
+ static long fastRoundDouble(double value) {
+ if (value >= 0) {
+ return (long) ((value + MAGIC_DOUBLE) - MAGIC_DOUBLE);
+ } else {
+ return (long) ((value - MAGIC_DOUBLE) + MAGIC_DOUBLE);
+ }
+ }
+
+ static long encodeDouble(double value, int exponent, int factor) {
+ return fastRoundDouble(value * DOUBLE_POW10[exponent] * DOUBLE_POW10_NEGATIVE[factor]);
+ }
+
+ static double decodeDouble(long encoded, int exponent, int factor) {
+ return encoded * DOUBLE_POW10[factor] * DOUBLE_POW10_NEGATIVE[exponent];
+ }
+
+ /**
+ * Number of bits needed to represent maxDelta as an unsigned value. This is the long counterpart
+ * to {@link org.apache.parquet.bytes.BytesUtils#getWidthFromMaxInt}, which only handles ints.
+ */
+ static int bitWidthForLong(long maxDelta) {
+ if (maxDelta == 0) {
+ return 0;
+ }
+ return Long.SIZE - Long.numberOfLeadingZeros(maxDelta);
+ }
+
+ public static class EncodingParams {
+ final int exponent;
+ final int factor;
+ final int numExceptions;
+
+ EncodingParams(int exponent, int factor, int numExceptions) {
+ this.exponent = exponent;
+ this.factor = factor;
+ this.numExceptions = numExceptions;
+ }
+ }
+
+ // Index positions within an (exponent, factor) pair array.
+ private static final int E = 0;
+ private static final int F = 1;
+
+ // All valid (exponent, factor) pairs for the full search, precomputed in nested-loop order
+ // (e ascending, then f ascending) so the tie-break and early-exit behave exactly like an inline
+ // double loop. Reused across every vector to avoid per-call allocation.
+ private static final int[][] ALL_VALID_FLOAT_PAIRS = buildAllPairs(FLOAT_MAX_EXPONENT);
+ private static final int[][] ALL_VALID_DOUBLE_PAIRS = buildAllPairs(DOUBLE_MAX_EXPONENT);
+
+ private static int[][] buildAllPairs(int maxExponent) {
+ int count = (maxExponent + 1) * (maxExponent + 2) / 2;
+ int[][] pairs = new int[count][];
+ int idx = 0;
+ for (int e = 0; e <= maxExponent; e++) {
+ for (int f = 0; f <= e; f++) {
+ pairs[idx++] = new int[] {e, f};
+ }
+ }
+ return pairs;
+ }
+
+ /** Try all (exponent, factor) combos and pick the one with the smallest estimated compressed size. */
+ static EncodingParams findBestFloatParams(float[] values, int offset, int length) {
+ return pickBestFloat(values, offset, length, ALL_VALID_FLOAT_PAIRS);
+ }
+
+ /** Same as findBestFloatParams but only tries the cached preset combos. */
+ static EncodingParams findBestFloatParamsWithPresets(float[] values, int offset, int length, int[][] presets) {
+ return pickBestFloat(values, offset, length, presets);
+ }
+
+ /**
+ * Scores each (exponent, factor) pair by estimated compressed size and returns the best.
+ *
+ * Estimated size (in bits) = {@code length * bitWidth + exceptions * (Float.SIZE + Short.SIZE)},
+ * where bitWidth is the number of bits needed to represent the signed range (max - min) of
+ * non-exception encoded values after frame-of-reference subtraction, matching the writer's FOR
+ * packing. This produces better compression ratios than minimizing exception count alone. Ties in
+ * size are broken toward the higher exponent, then the higher factor. The first pair (in {@code
+ * pairs} order) that encodes every value into a single FOR value with zero exceptions wins
+ * outright. When no pair yields any non-exception values, {@code pairs[0]} is returned.
+ */
+ private static EncodingParams pickBestFloat(float[] values, int offset, int length, int[][] pairs) {
+ int bestExponent = pairs[0][E];
+ int bestFactor = pairs[0][F];
+ int bestExceptions = length;
+ long bestEstimatedSize = Long.MAX_VALUE;
+
+ for (int[] pair : pairs) {
+ int e = pair[E];
+ int f = pair[F];
+ int exceptions = 0;
+ int minEncoded = Integer.MAX_VALUE;
+ int maxEncoded = Integer.MIN_VALUE;
+ for (int i = 0; i < length; i++) {
+ float value = values[offset + i];
+ if (isFloatException(value, e, f)) {
+ exceptions++;
+ } else {
+ int encoded = encodeFloat(value, e, f);
+ if (encoded < minEncoded) minEncoded = encoded;
+ if (encoded > maxEncoded) maxEncoded = encoded;
+ }
+ }
+ int nonExceptions = length - exceptions;
+ if (nonExceptions == 0) continue;
+ // Signed subtraction gives the true FOR span; Long.numberOfLeadingZeros yields the correct
+ // unsigned bit width, and an overflowing large range is penalized with 64 bits. This matches
+ // the writer's FOR packing.
+ long delta = (nonExceptions < 2) ? 0 : ((long) maxEncoded - (long) minEncoded);
+ int bitsPerValue = (delta == 0) ? 0 : (64 - Long.numberOfLeadingZeros(delta));
+ long estimatedSize = (long) length * bitsPerValue + (long) exceptions * (Float.SIZE + Short.SIZE);
+ if (estimatedSize < bestEstimatedSize
+ || (estimatedSize == bestEstimatedSize
+ && (e > bestExponent || (e == bestExponent && f > bestFactor)))) {
+ bestEstimatedSize = estimatedSize;
+ bestExponent = e;
+ bestFactor = f;
+ bestExceptions = exceptions;
+ if (bestExceptions == 0 && bitsPerValue == 0) {
+ return new EncodingParams(bestExponent, bestFactor, 0);
+ }
+ }
+ }
+ return new EncodingParams(bestExponent, bestFactor, bestExceptions);
+ }
+
+ /** Try all (exponent, factor) combos and pick the one with the smallest estimated compressed size. */
+ static EncodingParams findBestDoubleParams(double[] values, int offset, int length) {
+ return pickBestDouble(values, offset, length, ALL_VALID_DOUBLE_PAIRS);
+ }
+
+ /** Same as findBestDoubleParams but only tries the cached preset combos. */
+ static EncodingParams findBestDoubleParamsWithPresets(double[] values, int offset, int length, int[][] presets) {
+ return pickBestDouble(values, offset, length, presets);
+ }
+
+ /** Double counterpart to {@link #pickBestFloat}; see it for the scoring and tie-break rules. */
+ private static EncodingParams pickBestDouble(double[] values, int offset, int length, int[][] pairs) {
+ int bestExponent = pairs[0][E];
+ int bestFactor = pairs[0][F];
+ int bestExceptions = length;
+ long bestEstimatedSize = Long.MAX_VALUE;
+
+ for (int[] pair : pairs) {
+ int e = pair[E];
+ int f = pair[F];
+ int exceptions = 0;
+ long minEncoded = Long.MAX_VALUE;
+ long maxEncoded = Long.MIN_VALUE;
+ for (int i = 0; i < length; i++) {
+ double value = values[offset + i];
+ if (isDoubleException(value, e, f)) {
+ exceptions++;
+ } else {
+ long encoded = encodeDouble(value, e, f);
+ if (encoded < minEncoded) minEncoded = encoded;
+ if (encoded > maxEncoded) maxEncoded = encoded;
+ }
+ }
+ int nonExceptions = length - exceptions;
+ if (nonExceptions == 0) continue;
+ long delta = (nonExceptions < 2) ? 0 : (maxEncoded - minEncoded);
+ int bitsPerValue = (delta == 0) ? 0 : (64 - Long.numberOfLeadingZeros(delta));
+ long estimatedSize = (long) length * bitsPerValue + (long) exceptions * (Double.SIZE + Short.SIZE);
+ if (estimatedSize < bestEstimatedSize
+ || (estimatedSize == bestEstimatedSize
+ && (e > bestExponent || (e == bestExponent && f > bestFactor)))) {
+ bestEstimatedSize = estimatedSize;
+ bestExponent = e;
+ bestFactor = f;
+ bestExceptions = exceptions;
+ if (bestExceptions == 0 && bitsPerValue == 0) {
+ return new EncodingParams(bestExponent, bestFactor, 0);
+ }
+ }
+ }
+ return new EncodingParams(bestExponent, bestFactor, bestExceptions);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReader.java
new file mode 100644
index 0000000000..8a4e0e18ad
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReader.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.*;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * Abstract base class for ALP values readers with lazy per-vector decoding.
+ *
+ *
Reads ALP-encoded values from the interleaved page layout:
+ *
+ * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐
+ * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │
+ * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │
+ * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘
+ *
+ *
+ * Each vector is decoded lazily on first access. Skipping values does not
+ * trigger decoding of intermediate vectors.
+ */
+abstract class AlpValuesReader extends ValuesReader {
+
+ protected int vectorSize;
+ protected int totalCount;
+ protected int numVectors;
+ protected int pageValueIndex;
+ protected int currentVectorNumber;
+
+ protected int[] vectorOffsets;
+ protected ByteBuffer vectorsData;
+ protected int offsetArraySize;
+
+ // Scratch buffer for exception positions within a vector; shared by both readers (int[] in each).
+ protected int[] excPositionsBuffer;
+
+ AlpValuesReader() {
+ this.pageValueIndex = 0;
+ this.totalCount = 0;
+ this.currentVectorNumber = -1;
+ }
+
+ @Override
+ public void initFromPage(int valuesCount, ByteBufferInputStream stream)
+ throws ParquetDecodingException, IOException {
+ ByteBuffer headerBuf = stream.slice(ALP_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+ int compressionMode = headerBuf.get() & 0xFF;
+ int integerEncoding = headerBuf.get() & 0xFF;
+ int logVectorSize = headerBuf.get() & 0xFF;
+ int numElements = headerBuf.getInt();
+
+ if (compressionMode != ALP_COMPRESSION_MODE) {
+ throw new ParquetDecodingException("Unsupported ALP compression mode: " + compressionMode);
+ }
+ if (integerEncoding != ALP_INTEGER_ENCODING_FOR) {
+ throw new ParquetDecodingException("Unsupported ALP integer encoding: " + integerEncoding);
+ }
+ if (logVectorSize < MIN_LOG_VECTOR_SIZE || logVectorSize > MAX_LOG_VECTOR_SIZE) {
+ throw new ParquetDecodingException("Invalid ALP log vector size: " + logVectorSize + ", must be between "
+ + MIN_LOG_VECTOR_SIZE + " and " + MAX_LOG_VECTOR_SIZE);
+ }
+ if (numElements < 0) {
+ throw new ParquetDecodingException("Invalid ALP element count: " + numElements);
+ }
+ // ALP's num_elements is the count of non-null values that went through encoding;
+ // valuesCount is the page row count, which is larger when the column has nulls.
+ // The two are equal only for required (non-null) columns.
+ if (numElements > valuesCount) {
+ throw new ParquetDecodingException(
+ "ALP header element count " + numElements + " exceeds page valuesCount " + valuesCount);
+ }
+
+ this.vectorSize = 1 << logVectorSize;
+ this.totalCount = numElements;
+ this.numVectors = (numElements + vectorSize - 1) / vectorSize;
+ this.pageValueIndex = 0;
+ this.currentVectorNumber = -1;
+
+ this.offsetArraySize = numVectors * Integer.BYTES;
+ ByteBuffer offsetBuf = stream.slice(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN);
+ this.vectorOffsets = new int[numVectors];
+ for (int v = 0; v < numVectors; v++) {
+ vectorOffsets[v] = offsetBuf.getInt();
+ }
+
+ // Slice remaining bytes into a 0-based view so decodeVector can use
+ // absolute get methods (vectorsData.get(pos)) directly.
+ int remainingBytes = (int) stream.available();
+ ByteBuffer rawSlice = stream.slice(remainingBytes);
+ this.vectorsData = rawSlice.slice().order(ByteOrder.LITTLE_ENDIAN);
+
+ allocateDecodedBuffer(vectorSize);
+ this.excPositionsBuffer = new int[vectorSize];
+ }
+
+ protected int getVectorLength(int vectorNumber) {
+ if (vectorNumber < numVectors - 1) {
+ return vectorSize;
+ }
+ // Last vector may be partial
+ int lastVectorLen = totalCount % vectorSize;
+ return lastVectorLen == 0 ? vectorSize : lastVectorLen;
+ }
+
+ // Offsets in the page are relative to the compression body (after header),
+ // but vectorsData starts after the offset array, so adjust.
+ protected int getVectorDataPosition(int vectorNumber) {
+ return vectorOffsets[vectorNumber] - offsetArraySize;
+ }
+
+ @Override
+ public void skip() {
+ skip(1);
+ }
+
+ @Override
+ public void skip(int n) {
+ if (n < 0 || pageValueIndex + n > totalCount) {
+ throw new ParquetDecodingException(String.format(
+ "Cannot skip this many elements. Current index: %d. Skip %d. Total count: %d",
+ pageValueIndex, n, totalCount));
+ }
+ pageValueIndex += n;
+ }
+
+ protected void ensureVectorDecoded() {
+ int vectorNumber = pageValueIndex / vectorSize;
+ if (vectorNumber != currentVectorNumber) {
+ decodeVector(vectorNumber);
+ currentVectorNumber = vectorNumber;
+ }
+ }
+
+ /**
+ * Decodes one vector: parses and validates the shared ALP/FOR header fields, delegates the
+ * type-specific numeric work (FOR read, bit-unpacking, decode loop, exception values) to hooks,
+ * and reads the shared exception-position list in between. Runs once per vector, so the virtual
+ * hook calls are off the per-value hot path.
+ */
+ protected final void decodeVector(int vectorNumber) {
+ int vectorLen = getVectorLength(vectorNumber);
+ int pos = getVectorDataPosition(vectorNumber);
+
+ int exponent = vectorsData.get(pos) & 0xFF;
+ int factor = vectorsData.get(pos + 1) & 0xFF;
+ int numExceptions = vectorsData.getShort(pos + 2) & 0xFFFF;
+ pos += ALP_INFO_SIZE;
+
+ if (exponent > maxExponent()) {
+ throw new ParquetDecodingException("Invalid ALP " + typeName() + " exponent " + exponent + " in vector "
+ + vectorNumber + ", max is " + maxExponent());
+ }
+ if (factor > exponent) {
+ throw new ParquetDecodingException("Invalid ALP " + typeName() + " factor " + factor + " > exponent "
+ + exponent + " in vector " + vectorNumber);
+ }
+ if (numExceptions > vectorLen) {
+ throw new ParquetDecodingException("Invalid ALP numExceptions " + numExceptions + " > vectorLen "
+ + vectorLen + " in vector " + vectorNumber);
+ }
+
+ pos = decodeBody(pos, vectorNumber, vectorLen, exponent, factor);
+
+ if (numExceptions > 0) {
+ pos = readExceptionPositions(pos, numExceptions, vectorLen);
+ applyExceptionValues(pos, numExceptions);
+ }
+ }
+
+ /** Reads {@code numExceptions} little-endian uint16 positions into {@link #excPositionsBuffer}. */
+ private int readExceptionPositions(int pos, int numExceptions, int vectorLen) {
+ for (int e = 0; e < numExceptions; e++) {
+ excPositionsBuffer[e] = vectorsData.getShort(pos) & 0xFFFF;
+ if (excPositionsBuffer[e] >= vectorLen) {
+ throw new ParquetDecodingException("ALP exception position " + excPositionsBuffer[e]
+ + " out of bounds for vectorLen " + vectorLen);
+ }
+ pos += Short.BYTES;
+ }
+ return pos;
+ }
+
+ protected abstract void allocateDecodedBuffer(int capacity);
+
+ /** The maximum exponent for this value type (used to validate the header). */
+ protected abstract int maxExponent();
+
+ /** The value type name ("float" / "double") used in decoding error messages. */
+ protected abstract String typeName();
+
+ /**
+ * Reads the FOR header and bit-packed body starting at {@code pos}, decodes every slot into the
+ * type-specific decoded buffer, and returns the position just past the packed body.
+ */
+ protected abstract int decodeBody(int pos, int vectorNumber, int vectorLen, int exponent, int factor);
+
+ /**
+ * Reads {@code numExceptions} original values starting at {@code pos} and writes them into the
+ * decoded buffer at the positions previously read into {@link #excPositionsBuffer}.
+ */
+ protected abstract void applyExceptionValues(int pos, int numExceptions);
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForDouble.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForDouble.java
new file mode 100644
index 0000000000..14a10c6066
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForDouble.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.*;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * ALP values reader for DOUBLE type with lazy per-vector decoding.
+ *
+ *
Reads ALP-encoded double values from the interleaved page layout.
+ * Each vector is decoded on first access using BytePackerForLong-based unpacking.
+ */
+public class AlpValuesReaderForDouble extends AlpValuesReader {
+
+ private double[] decodedValues;
+ private long[] deltasBuffer;
+ private final long[] unpackPadBuf = new long[PACK_GROUP_SIZE];
+ private byte[] unpackByteBuf;
+
+ public AlpValuesReaderForDouble() {
+ super();
+ }
+
+ @Override
+ protected void allocateDecodedBuffer(int capacity) {
+ this.decodedValues = new double[capacity];
+ this.deltasBuffer = new long[capacity];
+ this.unpackByteBuf = new byte[Long.SIZE]; // max bit width for long = 64 bytes
+ }
+
+ @Override
+ protected int maxExponent() {
+ return DOUBLE_MAX_EXPONENT;
+ }
+
+ @Override
+ protected String typeName() {
+ return "double";
+ }
+
+ @Override
+ public double readDouble() {
+ if (pageValueIndex >= totalCount) {
+ throw new ParquetDecodingException("ALP double data was already exhausted.");
+ }
+ ensureVectorDecoded();
+ int vectorSlot = pageValueIndex % vectorSize;
+ pageValueIndex++;
+ return decodedValues[vectorSlot];
+ }
+
+ @Override
+ protected int decodeBody(int pos, int vectorNumber, int vectorLen, int exponent, int factor) {
+ long frameOfReference = vectorsData.getLong(pos);
+ int bitWidth = vectorsData.get(pos + Long.BYTES) & 0xFF;
+ if (bitWidth > Long.SIZE) {
+ throw new ParquetDecodingException(
+ "Invalid ALP double bitWidth " + bitWidth + " > 64 in vector " + vectorNumber);
+ }
+ pos += DOUBLE_FOR_INFO_SIZE;
+
+ if (bitWidth > 0) {
+ pos = unpackLongsWithBytePacker(vectorsData, pos, deltasBuffer, vectorLen, bitWidth);
+ } else {
+ Arrays.fill(deltasBuffer, 0, vectorLen, 0L);
+ }
+
+ for (int i = 0; i < vectorLen; i++) {
+ long encoded = deltasBuffer[i] + frameOfReference;
+ decodedValues[i] = AlpEncoderDecoder.decodeDouble(encoded, exponent, factor);
+ }
+ return pos;
+ }
+
+ @Override
+ protected void applyExceptionValues(int pos, int numExceptions) {
+ for (int e = 0; e < numExceptions; e++) {
+ decodedValues[excPositionsBuffer[e]] = vectorsData.getDouble(pos);
+ pos += Double.BYTES;
+ }
+ }
+
+ private int unpackLongsWithBytePacker(ByteBuffer buf, int pos, long[] output, int count, int bitWidth) {
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth);
+ int numFullGroups = count / PACK_GROUP_SIZE;
+ int remaining = count % PACK_GROUP_SIZE;
+
+ for (int g = 0; g < numFullGroups; g++) {
+ packer.unpack8Values(buf, pos, output, g * PACK_GROUP_SIZE);
+ pos += bitWidth;
+ }
+
+ // Last group might have fewer than 8 values; zero-pad and unpack,
+ // but only advance pos by the actual bytes in the page.
+ if (remaining > 0) {
+ int totalPackedBytes = BytesUtils.paddedByteCountFromBits(count * bitWidth);
+ int alreadyRead = numFullGroups * bitWidth;
+ int partialBytes = totalPackedBytes - alreadyRead;
+
+ for (int i = 0; i < partialBytes; i++) {
+ unpackByteBuf[i] = buf.get(pos + i);
+ }
+ for (int i = partialBytes; i < bitWidth; i++) {
+ unpackByteBuf[i] = 0;
+ }
+
+ packer.unpack8Values(unpackByteBuf, 0, unpackPadBuf, 0);
+ System.arraycopy(unpackPadBuf, 0, output, numFullGroups * PACK_GROUP_SIZE, remaining);
+ pos += partialBytes;
+ }
+
+ return pos;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForFloat.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForFloat.java
new file mode 100644
index 0000000000..54c0b2a72a
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForFloat.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.*;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * ALP values reader for FLOAT type with lazy per-vector decoding.
+ *
+ *
Reads ALP-encoded float values from the interleaved page layout.
+ * Each vector is decoded on first access using BytePacker-based unpacking.
+ */
+public class AlpValuesReaderForFloat extends AlpValuesReader {
+
+ private float[] decodedValues;
+ private int[] deltasBuffer;
+ private final int[] unpackPadBuf = new int[PACK_GROUP_SIZE];
+ private byte[] unpackByteBuf;
+
+ public AlpValuesReaderForFloat() {
+ super();
+ }
+
+ @Override
+ protected void allocateDecodedBuffer(int capacity) {
+ this.decodedValues = new float[capacity];
+ this.deltasBuffer = new int[capacity];
+ this.unpackByteBuf = new byte[Integer.SIZE]; // max bit width for int = 32 bytes
+ }
+
+ @Override
+ protected int maxExponent() {
+ return FLOAT_MAX_EXPONENT;
+ }
+
+ @Override
+ protected String typeName() {
+ return "float";
+ }
+
+ @Override
+ public float readFloat() {
+ if (pageValueIndex >= totalCount) {
+ throw new ParquetDecodingException("ALP float data was already exhausted.");
+ }
+ ensureVectorDecoded();
+ int vectorSlot = pageValueIndex % vectorSize;
+ pageValueIndex++;
+ return decodedValues[vectorSlot];
+ }
+
+ @Override
+ protected int decodeBody(int pos, int vectorNumber, int vectorLen, int exponent, int factor) {
+ int frameOfReference = vectorsData.getInt(pos);
+ int bitWidth = vectorsData.get(pos + Integer.BYTES) & 0xFF;
+ if (bitWidth > Integer.SIZE) {
+ throw new ParquetDecodingException(
+ "Invalid ALP float bitWidth " + bitWidth + " > 32 in vector " + vectorNumber);
+ }
+ pos += FLOAT_FOR_INFO_SIZE;
+
+ if (bitWidth > 0) {
+ pos = unpackIntsWithBytePacker(vectorsData, pos, deltasBuffer, vectorLen, bitWidth);
+ } else {
+ Arrays.fill(deltasBuffer, 0, vectorLen, 0);
+ }
+
+ for (int i = 0; i < vectorLen; i++) {
+ int encoded = deltasBuffer[i] + frameOfReference;
+ decodedValues[i] = AlpEncoderDecoder.decodeFloat(encoded, exponent, factor);
+ }
+ return pos;
+ }
+
+ // Overwrite exception slots with their original float values
+ @Override
+ protected void applyExceptionValues(int pos, int numExceptions) {
+ for (int e = 0; e < numExceptions; e++) {
+ decodedValues[excPositionsBuffer[e]] = vectorsData.getFloat(pos);
+ pos += Float.BYTES;
+ }
+ }
+
+ /** Unpack bit-packed ints in groups of 8, returns position after packed data. */
+ private int unpackIntsWithBytePacker(ByteBuffer buf, int pos, int[] output, int count, int bitWidth) {
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ int numFullGroups = count / PACK_GROUP_SIZE;
+ int remaining = count % PACK_GROUP_SIZE;
+
+ for (int g = 0; g < numFullGroups; g++) {
+ packer.unpack8Values(buf, pos, output, g * PACK_GROUP_SIZE);
+ pos += bitWidth;
+ }
+
+ if (remaining > 0) {
+ int totalPackedBytes = BytesUtils.paddedByteCountFromBits(count * bitWidth);
+ int alreadyRead = numFullGroups * bitWidth;
+ int partialBytes = totalPackedBytes - alreadyRead;
+
+ for (int i = 0; i < partialBytes; i++) {
+ unpackByteBuf[i] = buf.get(pos + i);
+ }
+ for (int i = partialBytes; i < bitWidth; i++) {
+ unpackByteBuf[i] = 0;
+ }
+
+ packer.unpack8Values(unpackByteBuf, 0, unpackPadBuf, 0);
+ System.arraycopy(unpackPadBuf, 0, output, numFullGroups * PACK_GROUP_SIZE, remaining);
+ pos += partialBytes;
+ }
+
+ return pos;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java
new file mode 100644
index 0000000000..36961c8b71
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java
@@ -0,0 +1,647 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.*;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.parquet.bytes.ByteBufferAllocator;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.bytes.CapacityByteArrayOutputStream;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+
+/**
+ * ALP (Adaptive Lossless floating-Point) values writer.
+ *
+ *
ALP encoding converts floating-point values to integers using decimal scaling,
+ * then applies Frame of Reference encoding and bit-packing.
+ * Values that cannot be losslessly converted are stored as exceptions.
+ *
+ *
Writing is incremental: values are buffered in a fixed-size vector buffer,
+ * and each full vector is encoded and flushed to the output stream immediately.
+ * On {@link #getBytes()}, any remaining partial vector is flushed, and the
+ * final page bytes are assembled.
+ *
+ *
Interleaved Page Layout:
+ *
+ * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐
+ * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │
+ * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │
+ * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘
+ *
+ *
+ * Each vector contains interleaved:
+ * AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions + ExceptionValues
+ */
+public abstract class AlpValuesWriter extends ValuesWriter {
+
+ protected final int initialCapacity;
+ protected final int pageSize;
+ protected final ByteBufferAllocator allocator;
+ protected final int vectorSize;
+ protected final int logVectorSize;
+
+ AlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) {
+ AlpConstants.validateVectorSize(vectorSize);
+ this.initialCapacity = initialCapacity;
+ this.pageSize = pageSize;
+ this.allocator = allocator;
+ this.vectorSize = vectorSize;
+ this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize);
+ }
+
+ @Override
+ public Encoding getEncoding() {
+ return Encoding.ALP;
+ }
+
+ /** Float writer. Buffers one vector at a time, encodes and flushes when full. */
+ public static class FloatAlpValuesWriter extends AlpValuesWriter {
+ private final float[] vectorBuffer;
+ private int bufferCount;
+ private int totalCount;
+ private CapacityByteArrayOutputStream encodedVectors;
+ private final List vectorByteSizes;
+
+ // Preset caching: collect evenly-spaced sample vectors across the rowgroup,
+ // then build presets using estimated compressed size (matching C++ AlpSampler).
+ private int vectorsProcessed;
+ private int[][] cachedPresets;
+ // Winning (exponent, factor) pairs from sampled vectors, tallied later into the preset cache.
+ private final List sampledParams;
+ private final int rowgroupSampleJump;
+
+ // Reusable per-vector buffers
+ private final int[] encodedBuffer;
+ private final short[] excPosBuffer;
+ private final float[] excValBuffer;
+ private final byte[] metadataBuf;
+ private final byte[] packBuf;
+ private final int[] packPadBuf;
+
+ public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
+ this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE);
+ }
+
+ public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) {
+ super(initialCapacity, pageSize, allocator, vectorSize);
+ this.vectorBuffer = new float[vectorSize];
+ this.bufferCount = 0;
+ this.totalCount = 0;
+ this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
+ this.vectorByteSizes = new ArrayList<>();
+ this.vectorsProcessed = 0;
+ this.cachedPresets = null;
+ this.sampledParams = new ArrayList<>();
+ // Space samples evenly: one sample every jump vectors across the rowgroup.
+ // Math.max(1, ...) guards against very small rowgroups or large vector sizes.
+ this.rowgroupSampleJump =
+ Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize);
+ // Pre-allocate reusable buffers
+ this.encodedBuffer = new int[vectorSize];
+ this.excPosBuffer = new short[vectorSize];
+ this.excValBuffer = new float[vectorSize];
+ this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, FLOAT_FOR_INFO_SIZE)];
+ this.packBuf = new byte[Integer.SIZE]; // max bit width for int
+ this.packPadBuf = new int[PACK_GROUP_SIZE];
+ }
+
+ @Override
+ public void writeFloat(float v) {
+ vectorBuffer[bufferCount++] = v;
+ totalCount++;
+ if (bufferCount == vectorSize) {
+ encodeAndFlushVector(bufferCount);
+ bufferCount = 0;
+ }
+ }
+
+ private void encodeAndFlushVector(int vectorLen) {
+ // Sampling phase first (full search + collect evenly-spaced samples, then build the preset
+ // cache once enough are gathered); after the cache is built, later vectors take the else branch.
+ AlpEncoderDecoder.EncodingParams params;
+ if (cachedPresets == null) {
+ params = AlpEncoderDecoder.findBestFloatParams(vectorBuffer, 0, vectorLen);
+ // Collect one sample every rowgroupSampleJump vectors so that samples are
+ // evenly distributed across the rowgroup (matching C++ AlpSampler spacing).
+ if (vectorsProcessed % rowgroupSampleJump == 0
+ && sampledParams.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) {
+ sampledParams.add(new int[] {params.exponent, params.factor});
+ }
+ if (sampledParams.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) {
+ buildPresetCache();
+ }
+ } else {
+ params = AlpEncoderDecoder.findBestFloatParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets);
+ }
+ vectorsProcessed++;
+
+ int excIdx = 0;
+
+ // Exception slots still occupy one position each in the packed integer array, but their real
+ // value lives in the exception block. Fill those slots with a placeholder encoding: the first
+ // non-exception value's encoding if one exists, otherwise 0 when the whole vector is exceptions.
+ // Reusing an encoding already present in the vector avoids introducing a spurious minimum
+ // (e.g. 0) that would distort the FOR base and bit width. The reader overwrites exception slots
+ // from the exception payload, so the placeholder only affects FOR/bit width, not decoded values.
+ int placeholder = 0;
+ for (int i = 0; i < vectorLen; i++) {
+ if (!AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) {
+ placeholder = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor);
+ break;
+ }
+ }
+
+ int minValue = Integer.MAX_VALUE;
+ for (int i = 0; i < vectorLen; i++) {
+ if (AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) {
+ excPosBuffer[excIdx] = (short) i;
+ excValBuffer[excIdx] = vectorBuffer[i];
+ excIdx++;
+ encodedBuffer[i] = placeholder;
+ } else {
+ encodedBuffer[i] = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor);
+ }
+ if (encodedBuffer[i] < minValue) {
+ minValue = encodedBuffer[i];
+ }
+ }
+
+ // Subtract min so deltas start at 0, reducing bit width.
+ // The subtraction may wrap for large ranges but unsigned bits stay correct.
+ int maxDelta = 0;
+ for (int i = 0; i < vectorLen; i++) {
+ encodedBuffer[i] = encodedBuffer[i] - minValue;
+ if (Integer.compareUnsigned(encodedBuffer[i], maxDelta) > 0) {
+ maxDelta = encodedBuffer[i];
+ }
+ }
+
+ int bitWidth = BytesUtils.getWidthFromMaxInt(maxDelta);
+
+ long startSize = encodedVectors.size();
+
+ // AlpInfo: exponent(1) + factor(1) + numExceptions(2) — little-endian
+ metadataBuf[0] = (byte) params.exponent;
+ metadataBuf[1] = (byte) params.factor;
+ metadataBuf[2] = (byte) (params.numExceptions & 0xFF);
+ metadataBuf[3] = (byte) ((params.numExceptions >>> 8) & 0xFF);
+ encodedVectors.write(metadataBuf, 0, ALP_INFO_SIZE);
+
+ // ForInfo: frameOfReference(4) + bitWidth(1) — little-endian
+ metadataBuf[0] = (byte) (minValue & 0xFF);
+ metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF);
+ metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF);
+ metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF);
+ metadataBuf[4] = (byte) bitWidth;
+ encodedVectors.write(metadataBuf, 0, FLOAT_FOR_INFO_SIZE);
+
+ if (bitWidth > 0) {
+ packIntsWithBytePacker(encodedBuffer, vectorLen, bitWidth);
+ }
+
+ // Exception positions then values, written as separate blocks
+ if (params.numExceptions > 0) {
+ for (int i = 0; i < params.numExceptions; i++) {
+ int pos = excPosBuffer[i] & 0xFFFF;
+ metadataBuf[0] = (byte) (pos & 0xFF);
+ metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF);
+ encodedVectors.write(metadataBuf, 0, Short.BYTES);
+ }
+
+ for (int i = 0; i < params.numExceptions; i++) {
+ int bits = Float.floatToRawIntBits(excValBuffer[i]);
+ metadataBuf[0] = (byte) (bits & 0xFF);
+ metadataBuf[1] = (byte) ((bits >>> 8) & 0xFF);
+ metadataBuf[2] = (byte) ((bits >>> 16) & 0xFF);
+ metadataBuf[3] = (byte) ((bits >>> 24) & 0xFF);
+ encodedVectors.write(metadataBuf, 0, Float.BYTES);
+ }
+ }
+
+ vectorByteSizes.add((int) (encodedVectors.size() - startSize));
+ }
+
+ private void packIntsWithBytePacker(int[] values, int count, int bitWidth) {
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ int numFullGroups = count / PACK_GROUP_SIZE;
+ int remaining = count % PACK_GROUP_SIZE;
+
+ for (int g = 0; g < numFullGroups; g++) {
+ packer.pack8Values(values, g * PACK_GROUP_SIZE, packBuf, 0);
+ encodedVectors.write(packBuf, 0, bitWidth);
+ }
+
+ // Partial last group: pack 8 values (zero-padded), but only write
+ // ceil(count * bitWidth / 8) - alreadyWritten bytes per spec.
+ if (remaining > 0) {
+ System.arraycopy(values, numFullGroups * PACK_GROUP_SIZE, packPadBuf, 0, remaining);
+ for (int i = remaining; i < PACK_GROUP_SIZE; i++) {
+ packPadBuf[i] = 0;
+ }
+ packer.pack8Values(packPadBuf, 0, packBuf, 0);
+ int totalPackedBytes = BytesUtils.paddedByteCountFromBits(count * bitWidth);
+ int alreadyWritten = numFullGroups * bitWidth;
+ encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten);
+ }
+ }
+
+ private void buildPresetCache() {
+ // Tally the (e,f) winners collected during sampling by how many samples each won,
+ // then take the top MAX_PRESET_COMBINATIONS.
+ Map comboCounts = new HashMap<>();
+ for (int[] ef : sampledParams) {
+ long key = ((long) ef[0] << Integer.SIZE) | ef[1];
+ comboCounts.merge(key, 1, Integer::sum);
+ }
+ List> sorted = new ArrayList<>(comboCounts.entrySet());
+ sorted.sort(Comparator., Integer>comparing(Map.Entry::getValue)
+ .reversed());
+ int numPresets = Math.min(sorted.size(), MAX_PRESET_COMBINATIONS);
+ cachedPresets = new int[numPresets][2];
+ for (int i = 0; i < numPresets; i++) {
+ long key = sorted.get(i).getKey();
+ cachedPresets[i][0] = (int) (key >>> Integer.SIZE); // exponent
+ cachedPresets[i][1] = (int) key; // factor
+ }
+ sampledParams.clear();
+ }
+
+ @Override
+ public long getBufferedSize() {
+ // Encoded vectors so far + estimate for buffered values.
+ // 3 bytes/value ≈ ALP_INFO_SIZE(4) + FLOAT_FOR_INFO_SIZE(5) amortized over vectorSize(1024),
+ // plus ~1 bit/value for packed data at typical bit widths — rounds up to ~3 bytes.
+ return encodedVectors.size() + (long) bufferCount * 3;
+ }
+
+ @Override
+ public BytesInput getBytes() {
+ if (bufferCount > 0) {
+ encodeAndFlushVector(bufferCount);
+ bufferCount = 0;
+ }
+
+ int numVectors = vectorByteSizes.size();
+
+ // Assemble: [header 7B] [offset array 4B*N] [vector data...]
+ ByteBuffer header = ByteBuffer.allocate(ALP_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+ header.put((byte) ALP_COMPRESSION_MODE);
+ header.put((byte) ALP_INTEGER_ENCODING_FOR);
+ header.put((byte) logVectorSize);
+ header.putInt(totalCount);
+
+ if (totalCount == 0) {
+ return BytesInput.from(header.array());
+ }
+
+ int offsetArraySize = numVectors * Integer.BYTES;
+ ByteBuffer offsets = ByteBuffer.allocate(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN);
+ int currentOffset = offsetArraySize;
+ for (int v = 0; v < numVectors; v++) {
+ offsets.putInt(currentOffset);
+ currentOffset += vectorByteSizes.get(v);
+ }
+
+ return BytesInput.concat(
+ BytesInput.from(header.array()), BytesInput.from(offsets.array()), BytesInput.from(encodedVectors));
+ }
+
+ @Override
+ public void reset() {
+ bufferCount = 0;
+ totalCount = 0;
+ encodedVectors.reset();
+ vectorByteSizes.clear();
+ vectorsProcessed = 0;
+ // Preserve cachedPresets across page boundaries: the preset (exponent, factor) pairs
+ // reflect the column's data distribution and remain valid for subsequent pages.
+ // Clearing them on every reset forces a full parameter search from scratch on each page,
+ // which is the main source of write overhead. Only reset the sampling state.
+ sampledParams.clear();
+ }
+
+ @Override
+ public void close() {
+ encodedVectors.close();
+ }
+
+ @Override
+ public long getAllocatedSize() {
+ return (long) vectorBuffer.length * Float.BYTES + encodedVectors.getCapacity();
+ }
+
+ @Override
+ public String memUsageString(String prefix) {
+ return String.format(
+ "%s FloatAlpValuesWriter %d values, %d bytes allocated", prefix, totalCount, getAllocatedSize());
+ }
+ }
+
+ /** Double writer. Same structure as FloatAlpValuesWriter but uses longs. */
+ public static class DoubleAlpValuesWriter extends AlpValuesWriter {
+ private final double[] vectorBuffer;
+ private int bufferCount;
+ private int totalCount;
+ private CapacityByteArrayOutputStream encodedVectors;
+ private final List vectorByteSizes;
+
+ // Preset caching: collect evenly-spaced sample vectors across the rowgroup,
+ // then build presets using estimated compressed size (matching C++ AlpSampler).
+ private int vectorsProcessed;
+ private int[][] cachedPresets;
+ // Winning (exponent, factor) pairs from sampled vectors, tallied later into the preset cache.
+ private final List sampledParams;
+ private final int rowgroupSampleJump;
+
+ // Reusable per-vector buffers
+ private final long[] encodedBuffer;
+ private final short[] excPosBuffer;
+ private final double[] excValBuffer;
+ private final byte[] metadataBuf;
+ private final byte[] packBuf;
+ private final long[] packPadBuf;
+
+ public DoubleAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
+ this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE);
+ }
+
+ public DoubleAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) {
+ super(initialCapacity, pageSize, allocator, vectorSize);
+ this.vectorBuffer = new double[vectorSize];
+ this.bufferCount = 0;
+ this.totalCount = 0;
+ this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
+ this.vectorByteSizes = new ArrayList<>();
+ this.vectorsProcessed = 0;
+ this.cachedPresets = null;
+ this.sampledParams = new ArrayList<>();
+ this.rowgroupSampleJump =
+ Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize);
+ // Pre-allocate reusable buffers
+ this.encodedBuffer = new long[vectorSize];
+ this.excPosBuffer = new short[vectorSize];
+ this.excValBuffer = new double[vectorSize];
+ this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, DOUBLE_FOR_INFO_SIZE)];
+ this.packBuf = new byte[Long.SIZE]; // max bit width for long
+ this.packPadBuf = new long[PACK_GROUP_SIZE];
+ }
+
+ @Override
+ public void writeDouble(double v) {
+ vectorBuffer[bufferCount++] = v;
+ totalCount++;
+ if (bufferCount == vectorSize) {
+ encodeAndFlushVector(bufferCount);
+ bufferCount = 0;
+ }
+ }
+
+ private void encodeAndFlushVector(int vectorLen) {
+ // Sampling phase first (full search + collect evenly-spaced samples, then build the preset
+ // cache once enough are gathered); after the cache is built, later vectors take the else branch.
+ AlpEncoderDecoder.EncodingParams params;
+ if (cachedPresets == null) {
+ params = AlpEncoderDecoder.findBestDoubleParams(vectorBuffer, 0, vectorLen);
+ if (vectorsProcessed % rowgroupSampleJump == 0
+ && sampledParams.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) {
+ sampledParams.add(new int[] {params.exponent, params.factor});
+ }
+ if (sampledParams.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) {
+ buildPresetCache();
+ }
+ } else {
+ params = AlpEncoderDecoder.findBestDoubleParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets);
+ }
+ vectorsProcessed++;
+
+ int excIdx = 0;
+
+ // Exception slots still occupy one position each in the packed long array, but their real
+ // value lives in the exception block. Fill those slots with a placeholder encoding: the first
+ // non-exception value's encoding if one exists, otherwise 0 when the whole vector is exceptions.
+ // Reusing an encoding already present in the vector avoids introducing a spurious minimum
+ // (e.g. 0) that would distort the FOR base and bit width. The reader overwrites exception slots
+ // from the exception payload, so the placeholder only affects FOR/bit width, not decoded values.
+ long placeholder = 0;
+ for (int i = 0; i < vectorLen; i++) {
+ if (!AlpEncoderDecoder.isDoubleException(vectorBuffer[i], params.exponent, params.factor)) {
+ placeholder = AlpEncoderDecoder.encodeDouble(vectorBuffer[i], params.exponent, params.factor);
+ break;
+ }
+ }
+
+ long minValue = Long.MAX_VALUE;
+ for (int i = 0; i < vectorLen; i++) {
+ if (AlpEncoderDecoder.isDoubleException(vectorBuffer[i], params.exponent, params.factor)) {
+ excPosBuffer[excIdx] = (short) i;
+ excValBuffer[excIdx] = vectorBuffer[i];
+ excIdx++;
+ encodedBuffer[i] = placeholder;
+ } else {
+ encodedBuffer[i] = AlpEncoderDecoder.encodeDouble(vectorBuffer[i], params.exponent, params.factor);
+ }
+ if (encodedBuffer[i] < minValue) {
+ minValue = encodedBuffer[i];
+ }
+ }
+
+ long maxDelta = 0;
+ for (int i = 0; i < vectorLen; i++) {
+ encodedBuffer[i] = encodedBuffer[i] - minValue;
+ if (Long.compareUnsigned(encodedBuffer[i], maxDelta) > 0) {
+ maxDelta = encodedBuffer[i];
+ }
+ }
+
+ int bitWidth = AlpEncoderDecoder.bitWidthForLong(maxDelta);
+
+ long startSize = encodedVectors.size();
+
+ // AlpInfo: exponent(1) + factor(1) + numExceptions(2) — little-endian
+ metadataBuf[0] = (byte) params.exponent;
+ metadataBuf[1] = (byte) params.factor;
+ metadataBuf[2] = (byte) (params.numExceptions & 0xFF);
+ metadataBuf[3] = (byte) ((params.numExceptions >>> 8) & 0xFF);
+ encodedVectors.write(metadataBuf, 0, ALP_INFO_SIZE);
+
+ // ForInfo: frameOfReference(8) + bitWidth(1) — little-endian
+ metadataBuf[0] = (byte) (minValue & 0xFF);
+ metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF);
+ metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF);
+ metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF);
+ metadataBuf[4] = (byte) ((minValue >>> 32) & 0xFF);
+ metadataBuf[5] = (byte) ((minValue >>> 40) & 0xFF);
+ metadataBuf[6] = (byte) ((minValue >>> 48) & 0xFF);
+ metadataBuf[7] = (byte) ((minValue >>> 56) & 0xFF);
+ metadataBuf[8] = (byte) bitWidth;
+ encodedVectors.write(metadataBuf, 0, DOUBLE_FOR_INFO_SIZE);
+
+ if (bitWidth > 0) {
+ packLongsWithBytePacker(encodedBuffer, vectorLen, bitWidth);
+ }
+
+ if (params.numExceptions > 0) {
+ for (int i = 0; i < params.numExceptions; i++) {
+ int pos = excPosBuffer[i] & 0xFFFF;
+ metadataBuf[0] = (byte) (pos & 0xFF);
+ metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF);
+ encodedVectors.write(metadataBuf, 0, Short.BYTES);
+ }
+
+ for (int i = 0; i < params.numExceptions; i++) {
+ long bits = Double.doubleToRawLongBits(excValBuffer[i]);
+ metadataBuf[0] = (byte) (bits & 0xFF);
+ metadataBuf[1] = (byte) ((bits >>> 8) & 0xFF);
+ metadataBuf[2] = (byte) ((bits >>> 16) & 0xFF);
+ metadataBuf[3] = (byte) ((bits >>> 24) & 0xFF);
+ metadataBuf[4] = (byte) ((bits >>> 32) & 0xFF);
+ metadataBuf[5] = (byte) ((bits >>> 40) & 0xFF);
+ metadataBuf[6] = (byte) ((bits >>> 48) & 0xFF);
+ metadataBuf[7] = (byte) ((bits >>> 56) & 0xFF);
+ encodedVectors.write(metadataBuf, 0, Double.BYTES);
+ }
+ }
+
+ vectorByteSizes.add((int) (encodedVectors.size() - startSize));
+ }
+
+ private void packLongsWithBytePacker(long[] values, int count, int bitWidth) {
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth);
+ int numFullGroups = count / PACK_GROUP_SIZE;
+ int remaining = count % PACK_GROUP_SIZE;
+
+ for (int g = 0; g < numFullGroups; g++) {
+ packer.pack8Values(values, g * PACK_GROUP_SIZE, packBuf, 0);
+ encodedVectors.write(packBuf, 0, bitWidth);
+ }
+
+ if (remaining > 0) {
+ System.arraycopy(values, numFullGroups * PACK_GROUP_SIZE, packPadBuf, 0, remaining);
+ for (int i = remaining; i < PACK_GROUP_SIZE; i++) {
+ packPadBuf[i] = 0;
+ }
+ packer.pack8Values(packPadBuf, 0, packBuf, 0);
+ int totalPackedBytes = BytesUtils.paddedByteCountFromBits(count * bitWidth);
+ int alreadyWritten = numFullGroups * bitWidth;
+ encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten);
+ }
+ }
+
+ private void buildPresetCache() {
+ Map comboCounts = new HashMap<>();
+ for (int[] ef : sampledParams) {
+ long key = ((long) ef[0] << Integer.SIZE) | ef[1];
+ comboCounts.merge(key, 1, Integer::sum);
+ }
+ List> sorted = new ArrayList<>(comboCounts.entrySet());
+ sorted.sort(Comparator., Integer>comparing(Map.Entry::getValue)
+ .reversed());
+ int numPresets = Math.min(sorted.size(), MAX_PRESET_COMBINATIONS);
+ cachedPresets = new int[numPresets][2];
+ for (int i = 0; i < numPresets; i++) {
+ long key = sorted.get(i).getKey();
+ cachedPresets[i][0] = (int) (key >>> Integer.SIZE);
+ cachedPresets[i][1] = (int) key;
+ }
+ sampledParams.clear();
+ }
+
+ @Override
+ public long getBufferedSize() {
+ // Encoded vectors so far + estimate for buffered values.
+ // 5 bytes/value ≈ ALP_INFO_SIZE(4) + DOUBLE_FOR_INFO_SIZE(9) amortized over vectorSize(1024),
+ // plus ~1 bit/value for packed data at typical bit widths — rounds up to ~5 bytes.
+ return encodedVectors.size() + (long) bufferCount * 5;
+ }
+
+ @Override
+ public BytesInput getBytes() {
+ if (bufferCount > 0) {
+ encodeAndFlushVector(bufferCount);
+ bufferCount = 0;
+ }
+
+ int numVectors = vectorByteSizes.size();
+
+ ByteBuffer header = ByteBuffer.allocate(ALP_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+ header.put((byte) ALP_COMPRESSION_MODE);
+ header.put((byte) ALP_INTEGER_ENCODING_FOR);
+ header.put((byte) logVectorSize);
+ header.putInt(totalCount);
+
+ if (totalCount == 0) {
+ return BytesInput.from(header.array());
+ }
+
+ // Offset array lets the reader jump to any vector in O(1)
+ int offsetArraySize = numVectors * Integer.BYTES;
+ ByteBuffer offsets = ByteBuffer.allocate(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN);
+ int currentOffset = offsetArraySize;
+ for (int v = 0; v < numVectors; v++) {
+ offsets.putInt(currentOffset);
+ currentOffset += vectorByteSizes.get(v);
+ }
+
+ return BytesInput.concat(
+ BytesInput.from(header.array()), BytesInput.from(offsets.array()), BytesInput.from(encodedVectors));
+ }
+
+ @Override
+ public void reset() {
+ bufferCount = 0;
+ totalCount = 0;
+ encodedVectors.reset();
+ vectorByteSizes.clear();
+ vectorsProcessed = 0;
+ // Preserve cachedPresets across page boundaries: the preset (exponent, factor) pairs
+ // reflect the column's data distribution and remain valid for subsequent pages.
+ // Clearing them on every reset forces a full parameter search from scratch on each page,
+ // which is the main source of write overhead. Only reset the sampling state.
+ sampledParams.clear();
+ }
+
+ @Override
+ public void close() {
+ encodedVectors.close();
+ }
+
+ @Override
+ public long getAllocatedSize() {
+ return (long) vectorBuffer.length * Double.BYTES + encodedVectors.getCapacity();
+ }
+
+ @Override
+ public String memUsageString(String prefix) {
+ return String.format(
+ "%s DoubleAlpValuesWriter %d values, %d bytes allocated", prefix, totalCount, getAllocatedSize());
+ }
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java
index e0d12c2878..cd2d329f1f 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java
@@ -24,6 +24,7 @@
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.alp.AlpValuesWriter;
import org.apache.parquet.column.values.bytestreamsplit.ByteStreamSplitValuesWriter;
import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter;
import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter;
@@ -147,7 +148,13 @@ private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) {
private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) {
final ValuesWriter fallbackWriter;
- if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
+ if (this.parquetProperties.isAlpEnabled(path)) {
+ fallbackWriter = new AlpValuesWriter.DoubleAlpValuesWriter(
+ parquetProperties.getInitialSlabSize(),
+ parquetProperties.getPageSizeThreshold(),
+ parquetProperties.getAllocator(),
+ parquetProperties.getAlpVectorSize(path));
+ } else if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
fallbackWriter = new ByteStreamSplitValuesWriter.DoubleByteStreamSplitValuesWriter(
parquetProperties.getInitialSlabSize(),
parquetProperties.getPageSizeThreshold(),
@@ -164,7 +171,13 @@ private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) {
private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) {
final ValuesWriter fallbackWriter;
- if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
+ if (this.parquetProperties.isAlpEnabled(path)) {
+ fallbackWriter = new AlpValuesWriter.FloatAlpValuesWriter(
+ parquetProperties.getInitialSlabSize(),
+ parquetProperties.getPageSizeThreshold(),
+ parquetProperties.getAllocator(),
+ parquetProperties.getAlpVectorSize(path));
+ } else if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
fallbackWriter = new ByteStreamSplitValuesWriter.FloatByteStreamSplitValuesWriter(
parquetProperties.getInitialSlabSize(),
parquetProperties.getPageSizeThreshold(),
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java
index c50b4e49c5..9bfe72b4f8 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java
@@ -25,6 +25,7 @@
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.alp.AlpValuesWriter;
import org.apache.parquet.column.values.bytestreamsplit.ByteStreamSplitValuesWriter;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong;
@@ -159,7 +160,13 @@ private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) {
private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) {
final ValuesWriter fallbackWriter;
- if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
+ if (this.parquetProperties.isAlpEnabled(path)) {
+ fallbackWriter = new AlpValuesWriter.DoubleAlpValuesWriter(
+ parquetProperties.getInitialSlabSize(),
+ parquetProperties.getPageSizeThreshold(),
+ parquetProperties.getAllocator(),
+ parquetProperties.getAlpVectorSize(path));
+ } else if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
fallbackWriter = new ByteStreamSplitValuesWriter.DoubleByteStreamSplitValuesWriter(
parquetProperties.getInitialSlabSize(),
parquetProperties.getPageSizeThreshold(),
@@ -176,7 +183,13 @@ private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) {
private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) {
final ValuesWriter fallbackWriter;
- if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
+ if (this.parquetProperties.isAlpEnabled(path)) {
+ fallbackWriter = new AlpValuesWriter.FloatAlpValuesWriter(
+ parquetProperties.getInitialSlabSize(),
+ parquetProperties.getPageSizeThreshold(),
+ parquetProperties.getAllocator(),
+ parquetProperties.getAlpVectorSize(path));
+ } else if (this.parquetProperties.isByteStreamSplitEnabled(path)) {
fallbackWriter = new ByteStreamSplitValuesWriter.FloatByteStreamSplitValuesWriter(
parquetProperties.getInitialSlabSize(),
parquetProperties.getPageSizeThreshold(),
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpAdversarialTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpAdversarialTest.java
new file mode 100644
index 0000000000..c681bf8c47
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpAdversarialTest.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.junit.Test;
+
+/**
+ * Adversarial tests for ALP readers: feed malformed page bytes and assert the reader
+ * fails cleanly rather than crashing, producing silent garbage, or hanging.
+ *
+ * "Fails cleanly" means raising a meaningful exception — preferably
+ * {@link ParquetDecodingException}, but at minimum a typed exception (not a JVM-level
+ * crash, infinite loop, or wrong answer). The tests cover both:
+ *
+ * - Already-validated cases — the reader explicitly rejects these with a
+ * ParquetDecodingException carrying an explanatory message. These tests pin
+ * the validation behavior in place.
+ *
- Currently-unvalidated cases (truncation, corrupted offsets) — the reader
+ * relies on the underlying ByteBuffer to surface IndexOutOfBoundsException or
+ * BufferUnderflowException. These tests assert that some Throwable is raised
+ * so the failure mode stays "loud" even if the explicit message is missing.
+ *
+ */
+public class AlpAdversarialTest {
+
+ // ---------------------------------------------------------------------------
+ // Helpers: build a known-good encoded page, then mutate copies of it
+ // ---------------------------------------------------------------------------
+
+ /** Build a valid ALP-encoded double page with N clean values. */
+ private static byte[] validDoublePage(int valueCount, int vectorSize) throws Exception {
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ int cap = Math.max(512, valueCount * 16);
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(cap, cap, new DirectByteBufferAllocator(), vectorSize);
+ // 2-decimal values — the ALP sweet spot, ensures no exceptions
+ for (int i = 0; i < valueCount; i++) {
+ writer.writeDouble((i % 1000) / 100.0);
+ }
+ BytesInput bi = writer.getBytes();
+ ByteBuffer bb = bi.toByteBuffer();
+ byte[] out = new byte[bb.remaining()];
+ bb.duplicate().get(out);
+ return out;
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ /** Build a valid ALP-encoded float page with N clean values. */
+ private static byte[] validFloatPage(int valueCount, int vectorSize) throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ int cap = Math.max(256, valueCount * 8);
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(cap, cap, new DirectByteBufferAllocator(), vectorSize);
+ for (int i = 0; i < valueCount; i++) {
+ writer.writeFloat((i % 1000) / 100.0f);
+ }
+ BytesInput bi = writer.getBytes();
+ ByteBuffer bb = bi.toByteBuffer();
+ byte[] out = new byte[bb.remaining()];
+ bb.duplicate().get(out);
+ return out;
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ /** Sanity baseline: the known-good page actually decodes cleanly. */
+ @Test
+ public void sanityBaselineDecodesClean() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ for (int i = 0; i < 32; i++) reader.readDouble();
+ }
+
+ // ---------------------------------------------------------------------------
+ // Header-level validation (already-validated paths)
+ // ---------------------------------------------------------------------------
+
+ @Test
+ public void rejectsBadCompressionMode() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[0] = (byte) 0x99; // mode is at byte 0
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, () -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ });
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("compression"));
+ }
+
+ @Test
+ public void rejectsBadIntegerEncoding() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[1] = (byte) 0x99; // integer_encoding is at byte 1
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, () -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ });
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("integer encoding"));
+ }
+
+ @Test
+ public void rejectsLogVectorSizeTooLarge() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[2] = (byte) 99; // log_vector_size at byte 2
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, () -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ });
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("vector size"));
+ }
+
+ @Test
+ public void rejectsLogVectorSizeTooSmall() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[2] = (byte) 2; // below MIN_LOG_VECTOR_SIZE=3
+ assertThrows(ParquetDecodingException.class, () -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ });
+ }
+
+ @Test
+ public void rejectsNegativeNumElements() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // num_elements is int32 LE at bytes 3..6 — write -1
+ page[3] = (byte) 0xFF;
+ page[4] = (byte) 0xFF;
+ page[5] = (byte) 0xFF;
+ page[6] = (byte) 0xFF;
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, () -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ });
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("element count"));
+ }
+
+ @Test
+ public void rejectsNumElementsGreaterThanValuesCount() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // num_elements stays 32; pass valuesCount=10 (smaller than encoded count)
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, () -> {
+ new AlpValuesReaderForDouble().initFromPage(10, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ });
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("exceeds"));
+ }
+
+ // ---------------------------------------------------------------------------
+ // Vector-level validation (already-validated paths, surface lazily on decode)
+ // ---------------------------------------------------------------------------
+
+ /** Helper: find the byte position where the first vector's metadata starts. */
+ private static int firstVectorOffset(byte[] page) {
+ // header (7) + first 4 bytes of offset array = the offset value itself
+ int firstVectorOff =
+ ByteBuffer.wrap(page, 7, 4).order(ByteOrder.LITTLE_ENDIAN).getInt();
+ // offsets are measured from the start of the compression body (after the 7B header)
+ return 7 + firstVectorOff;
+ }
+
+ @Test
+ public void rejectsExponentTooHighDouble() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0] = (byte) 99; // exponent byte
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readDouble);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("exponent"));
+ }
+
+ @Test
+ public void rejectsExponentTooHighFloat() throws Exception {
+ byte[] page = validFloatPage(32, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0] = (byte) 99;
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readFloat);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("exponent"));
+ }
+
+ @Test
+ public void rejectsFactorGreaterThanExponent() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0] = (byte) 2; // exponent
+ page[v0 + 1] = (byte) 5; // factor > exponent
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readDouble);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("factor"));
+ }
+
+ @Test
+ public void rejectsTooManyExceptions() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ // num_exceptions at v0+2, uint16 LE — set to 9999, way more than vectorLen=16
+ page[v0 + 2] = (byte) (9999 & 0xFF);
+ page[v0 + 3] = (byte) ((9999 >>> 8) & 0xFF);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readDouble);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("numexceptions"));
+ }
+
+ @Test
+ public void rejectsBitWidthTooLargeDouble() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ // Layout: ALP_INFO(4) + frameOfReference(8) then bitWidth byte at v0+12. 99 > 64.
+ page[v0 + 12] = (byte) 99;
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readDouble);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("bitwidth"));
+ }
+
+ @Test
+ public void rejectsBitWidthTooLargeFloat() throws Exception {
+ byte[] page = validFloatPage(32, 16);
+ int v0 = firstVectorOffset(page);
+ // Layout: ALP_INFO(4) + frameOfReference(4) then bitWidth byte at v0+8. 99 > 32.
+ page[v0 + 8] = (byte) 99;
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readFloat);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("bitwidth"));
+ }
+
+ // ---------------------------------------------------------------------------
+ // Currently-unvalidated paths: truncation and corrupted offsets
+ // These currently fail with low-level Throwables (BufferUnderflowException,
+ // IndexOutOfBoundsException). The tests assert any Throwable is raised so we
+ // notice if a regression silently swallows the corruption.
+ // ---------------------------------------------------------------------------
+
+ /** Page with only the 7-byte header — nothing else. */
+ @Test
+ public void rejectsHeaderOnlyPage() {
+ byte[] tiny = new byte[] {0x00, 0x00, 0x0A, 0x20, 0x00, 0x00, 0x00}; // 32 elements, log_vec=10
+ Throwable t = catchAny(() -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(tiny)));
+ });
+ assertNotNull("header-only page must raise", t);
+ }
+
+ @Test
+ public void rejectsPageTruncatedMidOffsetArray() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // num_vectors = ceil(32/16) = 2, so offset array is 8 bytes. Truncate to chop the 2nd offset.
+ byte[] truncated = new byte[7 + 4]; // header + first offset only
+ System.arraycopy(page, 0, truncated, 0, truncated.length);
+ Throwable t = catchAny(() -> {
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(truncated)));
+ });
+ assertNotNull("truncated offset array must raise", t);
+ }
+
+ @Test
+ public void rejectsPageTruncatedMidVectorData() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // chop the last 20 bytes — guaranteed to land in the middle of the second vector
+ byte[] truncated = new byte[page.length - 20];
+ System.arraycopy(page, 0, truncated, 0, truncated.length);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ // initFromPage should still succeed (truncation is inside the vectors section,
+ // which initFromPage just slices without parsing). The failure surfaces on decode.
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(truncated)));
+ Throwable t = catchAny(() -> {
+ for (int i = 0; i < 32; i++) reader.readDouble();
+ });
+ assertNotNull("truncated vector data must raise on read", t);
+ }
+
+ @Test
+ public void rejectsCorruptedOffsetPointingPastEnd() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // Offset array starts at byte 7. Overwrite the first offset (uint32 LE) with a huge value.
+ page[7] = (byte) 0xFF;
+ page[8] = (byte) 0xFF;
+ page[9] = (byte) 0xFF;
+ page[10] = (byte) 0x7F;
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ Throwable t = catchAny(() -> reader.readDouble());
+ assertNotNull("corrupted offset must raise on decode", t);
+ }
+
+ // ---------------------------------------------------------------------------
+ // skip() / read() bounds
+ // ---------------------------------------------------------------------------
+
+ @Test
+ public void rejectsSkipPastEnd() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThrows(ParquetDecodingException.class, () -> reader.skip(33));
+ }
+
+ @Test
+ public void rejectsNegativeSkip() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThrows(ParquetDecodingException.class, () -> reader.skip(-1));
+ }
+
+ @Test
+ public void rejectsReadPastEnd() throws Exception {
+ byte[] page = validDoublePage(8, 8);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(8, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ for (int i = 0; i < 8; i++) reader.readDouble();
+ ParquetDecodingException ex = assertThrows(ParquetDecodingException.class, reader::readDouble);
+ assertTrue(ex.getMessage(), ex.getMessage().toLowerCase().contains("exhausted"));
+ }
+
+ // ---------------------------------------------------------------------------
+ // Utility
+ // ---------------------------------------------------------------------------
+
+ @FunctionalInterface
+ private interface ThrowingRunnable {
+ void run() throws Throwable;
+ }
+
+ /** Catch any Throwable (including low-level RuntimeExceptions / Errors). */
+ private static Throwable catchAny(ThrowingRunnable r) {
+ try {
+ r.run();
+ } catch (Throwable t) {
+ return t;
+ }
+ fail("Expected a Throwable but none was raised");
+ return null; // unreachable
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpBitPackingTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpBitPackingTest.java
new file mode 100644
index 0000000000..9660bcc7e2
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpBitPackingTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.junit.Test;
+
+/**
+ * Tests for bit-packing behavior in the ALP encoding pipeline.
+ */
+public class AlpBitPackingTest {
+
+ @Test
+ public void testBytePackerIntRoundTrip() {
+ // Verify BytePacker pack/unpack round-trip for every bit width, including the
+ // full 32-bit width where the frame-of-reference delta uses all int bits.
+ for (int bitWidth = 1; bitWidth <= 32; bitWidth++) {
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ // Largest value representable in bitWidth bits (unsigned). At width 32 that is
+ // all bits set (-1); (1L << 32) - 1 would not fit in an int, so special-case it.
+ int maxVal = (bitWidth == 32) ? -1 : (int) ((1L << bitWidth) - 1);
+
+ int[] input = new int[8];
+ for (int i = 0; i < 7; i++) {
+ input[i] = (maxVal >>> 3) * i; // spread of values, each fitting in bitWidth bits
+ }
+ input[7] = maxVal; // exercise the top bit of this width
+
+ byte[] packed = new byte[bitWidth];
+ packer.pack8Values(input, 0, packed, 0);
+
+ int[] unpacked = new int[8];
+ ByteBuffer buf = ByteBuffer.wrap(packed);
+ packer.unpack8Values(buf, 0, unpacked, 0);
+
+ for (int i = 0; i < 8; i++) {
+ assertEquals("BitWidth=" + bitWidth + " index=" + i, input[i], unpacked[i]);
+ }
+ }
+ }
+
+ @Test
+ public void testBytePackerForLongRoundTrip() {
+ // Verify BytePackerForLong pack/unpack round-trip for every bit width, including the
+ // full 64-bit width where the frame-of-reference delta uses all long bits.
+ for (int bitWidth = 1; bitWidth <= 64; bitWidth++) {
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth);
+ // Largest value representable in bitWidth bits (unsigned). At width 64 that is
+ // all bits set (-1L); (1L << 64) - 1 would wrap to 0, so special-case it.
+ long maxVal = (bitWidth == 64) ? -1L : (1L << bitWidth) - 1;
+
+ long[] input = new long[8];
+ for (int i = 0; i < 7; i++) {
+ input[i] = (maxVal >>> 3) * i; // spread of values, each fitting in bitWidth bits
+ }
+ input[7] = maxVal; // exercise the top bit of this width
+
+ byte[] packed = new byte[bitWidth];
+ packer.pack8Values(input, 0, packed, 0);
+
+ long[] unpacked = new long[8];
+ ByteBuffer buf = ByteBuffer.wrap(packed);
+ packer.unpack8Values(buf, 0, unpacked, 0);
+
+ for (int i = 0; i < 8; i++) {
+ assertEquals("BitWidth=" + bitWidth + " index=" + i, input[i], unpacked[i]);
+ }
+ }
+ }
+
+ @Test
+ public void testSimpleTwoFloats() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(2, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertEquals(Float.floatToRawIntBits(1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ assertEquals(Float.floatToRawIntBits(2.0f), Float.floatToRawIntBits(reader.readFloat()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testThreeFloatsWithNegative() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(-1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(3, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertEquals(Float.floatToRawIntBits(1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ assertEquals(Float.floatToRawIntBits(-1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ assertEquals(Float.floatToRawIntBits(2.0f), Float.floatToRawIntBits(reader.readFloat()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testEncoderDecoderDirectly() {
+ float value = 1.0f;
+ int exponent = 2;
+ int factor = 0;
+
+ assertFalse(AlpEncoderDecoder.isFloatException(value, exponent, factor));
+ int encoded = AlpEncoderDecoder.encodeFloat(value, exponent, factor);
+ float decoded = AlpEncoderDecoder.decodeFloat(encoded, exponent, factor);
+ assertEquals(Float.floatToRawIntBits(value), Float.floatToRawIntBits(decoded));
+ }
+
+ @Test
+ public void testHeaderFormatValidation() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ byte[] bytes = input.toByteArray();
+
+ // Parse and validate header (7 bytes: mode + encoding + logVectorSize + count)
+ ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+ int compressionMode = buf.get() & 0xFF;
+ int integerEncoding = buf.get() & 0xFF;
+ int logVectorSize = buf.get() & 0xFF;
+ int numElements = buf.getInt();
+
+ assertEquals(AlpConstants.ALP_COMPRESSION_MODE, compressionMode);
+ assertEquals(AlpConstants.ALP_INTEGER_ENCODING_FOR, integerEncoding);
+ assertEquals(AlpConstants.DEFAULT_VECTOR_SIZE_LOG, logVectorSize);
+ assertEquals(2, numElements);
+
+ // Verify offset array follows header (1 vector = 1 offset entry)
+ int offset0 = buf.getInt();
+ // First vector starts right after the offset array (1 vector * 4 bytes = 4)
+ assertEquals(Integer.BYTES, offset0);
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testPartialGroupPacking() throws Exception {
+ // Test with fewer than 8 values to exercise partial group packing/unpacking
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ float[] values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f};
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (float expected : values) {
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testBitWidthZeroPacking() throws Exception {
+ // All identical values should result in bitWidth=0 (no packed data)
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ for (int i = 0; i < 10; i++) {
+ writer.writeFloat(5.0f);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(10, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < 10; i++) {
+ assertEquals(Float.floatToRawIntBits(5.0f), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDoublePackingRoundTrip() throws Exception {
+ // Verify double packing round-trip with varying values
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(512, 512, new DirectByteBufferAllocator());
+ double[] values = {1.0, -1.0, 2.0, 100.5, 0.0, 3.14, 42.0, 99.99, 0.001, 7.77};
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (double expected : values) {
+ assertEquals(Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(reader.readDouble()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testExactEightValues() throws Exception {
+ // Exactly 8 values = one full packing group, no partial group
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ float[] values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f};
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (float expected : values) {
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpEncoderDecoderTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpEncoderDecoderTest.java
new file mode 100644
index 0000000000..a8063b1e52
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpEncoderDecoderTest.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+/**
+ * Tests for the core ALP encoder/decoder logic.
+ */
+public class AlpEncoderDecoderTest {
+
+ // ========== Float Encoding/Decoding Tests ==========
+
+ @Test
+ public void testFloatRoundTrip() {
+ float[] testValues = {0.0f, 1.0f, -1.0f, 3.14159f, 100.5f, 0.001f, 1234567.0f};
+
+ for (float value : testValues) {
+ for (int exponent = 0; exponent <= AlpConstants.FLOAT_MAX_EXPONENT; exponent++) {
+ for (int factor = 0; factor <= exponent; factor++) {
+ if (!AlpEncoderDecoder.isFloatException(value, exponent, factor)) {
+ int encoded = AlpEncoderDecoder.encodeFloat(value, exponent, factor);
+ float decoded = AlpEncoderDecoder.decodeFloat(encoded, exponent, factor);
+ assertEquals(
+ "Round-trip failed for value=" + value + ", exponent=" + exponent + ", factor="
+ + factor,
+ Float.floatToRawIntBits(value),
+ Float.floatToRawIntBits(decoded));
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testFloatExceptionDetection() {
+ assertTrue("NaN should be an exception", AlpEncoderDecoder.isIntrinsicFloatException(Float.NaN));
+ assertTrue(
+ "Positive infinity should be an exception",
+ AlpEncoderDecoder.isIntrinsicFloatException(Float.POSITIVE_INFINITY));
+ assertTrue(
+ "Negative infinity should be an exception",
+ AlpEncoderDecoder.isIntrinsicFloatException(Float.NEGATIVE_INFINITY));
+ assertTrue("Negative zero should be an exception", AlpEncoderDecoder.isIntrinsicFloatException(-0.0f));
+
+ assertFalse("1.0f should not be a basic exception", AlpEncoderDecoder.isIntrinsicFloatException(1.0f));
+ assertFalse("0.0f should not be a basic exception", AlpEncoderDecoder.isIntrinsicFloatException(0.0f));
+ }
+
+ @Test
+ public void testFloatEncoding() {
+ assertEquals(123, AlpEncoderDecoder.encodeFloat(1.23f, 2, 0));
+ assertEquals(123, AlpEncoderDecoder.encodeFloat(12.3f, 2, 1));
+ assertEquals(0, AlpEncoderDecoder.encodeFloat(0.0f, 5, 0));
+ }
+
+ @Test
+ public void testFloatDecoding() {
+ assertEquals(1.23f, AlpEncoderDecoder.decodeFloat(123, 2, 0), 1e-6f);
+ assertEquals(12.3f, AlpEncoderDecoder.decodeFloat(123, 2, 1), 1e-6f);
+ assertEquals(0.0f, AlpEncoderDecoder.decodeFloat(0, 5, 0), 0.0f);
+ }
+
+ @Test
+ public void testFloatEncodeRounding() {
+ // Verify rounding behavior (magic number trick rounds to nearest)
+ assertEquals(5, AlpEncoderDecoder.encodeFloat(5.4f, 0, 0));
+ assertEquals(6, AlpEncoderDecoder.encodeFloat(5.6f, 0, 0));
+ assertEquals(-5, AlpEncoderDecoder.encodeFloat(-5.4f, 0, 0));
+ assertEquals(-6, AlpEncoderDecoder.encodeFloat(-5.6f, 0, 0));
+ assertEquals(0, AlpEncoderDecoder.encodeFloat(0.0f, 0, 0));
+ }
+
+ @Test
+ public void testFloatEncodeDecodeWithFactor() {
+ // Verify that encode/decode with non-zero factor works correctly.
+ // The key correctness property: encode uses value * POW10[e] * POW10_NEGATIVE[f],
+ // and decode uses encoded * POW10[f] * POW10_NEGATIVE[e].
+ float value = 12.3f;
+ int encoded = AlpEncoderDecoder.encodeFloat(value, 2, 1);
+ assertEquals(123, encoded); // 12.3 * 100 * 0.1 = 123
+ float decoded = AlpEncoderDecoder.decodeFloat(encoded, 2, 1);
+ assertEquals(Float.floatToRawIntBits(value), Float.floatToRawIntBits(decoded));
+ }
+
+ // ========== Double Encoding/Decoding Tests ==========
+
+ @Test
+ public void testDoubleRoundTrip() {
+ double[] testValues = {0.0, 1.0, -1.0, 3.14159265358979, 100.5, 0.001, 12345678901234.0};
+
+ for (double value : testValues) {
+ for (int exponent = 0; exponent <= Math.min(AlpConstants.DOUBLE_MAX_EXPONENT, 10); exponent++) {
+ for (int factor = 0; factor <= exponent; factor++) {
+ if (!AlpEncoderDecoder.isDoubleException(value, exponent, factor)) {
+ long encoded = AlpEncoderDecoder.encodeDouble(value, exponent, factor);
+ double decoded = AlpEncoderDecoder.decodeDouble(encoded, exponent, factor);
+ assertEquals(
+ "Round-trip failed for value=" + value + ", exponent=" + exponent + ", factor="
+ + factor,
+ Double.doubleToRawLongBits(value),
+ Double.doubleToRawLongBits(decoded));
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleExceptionDetection() {
+ assertTrue("NaN should be an exception", AlpEncoderDecoder.isIntrinsicDoubleException(Double.NaN));
+ assertTrue(
+ "Positive infinity should be an exception",
+ AlpEncoderDecoder.isIntrinsicDoubleException(Double.POSITIVE_INFINITY));
+ assertTrue(
+ "Negative infinity should be an exception",
+ AlpEncoderDecoder.isIntrinsicDoubleException(Double.NEGATIVE_INFINITY));
+ assertTrue("Negative zero should be an exception", AlpEncoderDecoder.isIntrinsicDoubleException(-0.0));
+
+ assertFalse("1.0 should not be a basic exception", AlpEncoderDecoder.isIntrinsicDoubleException(1.0));
+ assertFalse("0.0 should not be a basic exception", AlpEncoderDecoder.isIntrinsicDoubleException(0.0));
+ }
+
+ @Test
+ public void testDoubleEncoding() {
+ assertEquals(123L, AlpEncoderDecoder.encodeDouble(1.23, 2, 0));
+ assertEquals(123L, AlpEncoderDecoder.encodeDouble(12.3, 2, 1));
+ assertEquals(0L, AlpEncoderDecoder.encodeDouble(0.0, 5, 0));
+ }
+
+ @Test
+ public void testDoubleDecoding() {
+ assertEquals(1.23, AlpEncoderDecoder.decodeDouble(123, 2, 0), 1e-10);
+ assertEquals(12.3, AlpEncoderDecoder.decodeDouble(123, 2, 1), 1e-10);
+ assertEquals(0.0, AlpEncoderDecoder.decodeDouble(0, 5, 0), 0.0);
+ }
+
+ @Test
+ public void testDoubleEncodeRounding() {
+ // Verify rounding behavior (magic number trick rounds to nearest)
+ assertEquals(5L, AlpEncoderDecoder.encodeDouble(5.4, 0, 0));
+ assertEquals(6L, AlpEncoderDecoder.encodeDouble(5.6, 0, 0));
+ assertEquals(-5L, AlpEncoderDecoder.encodeDouble(-5.4, 0, 0));
+ assertEquals(-6L, AlpEncoderDecoder.encodeDouble(-5.6, 0, 0));
+ assertEquals(0L, AlpEncoderDecoder.encodeDouble(0.0, 0, 0));
+ }
+
+ @Test
+ public void testDoubleEncodeDecodeWithFactor() {
+ // Verify that encode/decode with non-zero factor works correctly.
+ // The key correctness property: encode uses value * POW10[e] * POW10_NEGATIVE[f],
+ // and decode uses encoded * POW10[f] * POW10_NEGATIVE[e].
+ double value = 12.3;
+ long encoded = AlpEncoderDecoder.encodeDouble(value, 2, 1);
+ assertEquals(123L, encoded); // 12.3 * 100 * 0.1 = 123
+ double decoded = AlpEncoderDecoder.decodeDouble(encoded, 2, 1);
+ assertEquals(Double.doubleToRawLongBits(value), Double.doubleToRawLongBits(decoded));
+ }
+
+ @Test
+ public void testDoubleEncodeDecodeArithmeticOrder() {
+ // This test verifies that the exact order of operations in encode/decode
+ // is critical for IEEE 754 correctness. The encode uses
+ // fastRound(value * POW10[e] * POW10_NEGATIVE[f]) and decode uses
+ // (encoded * POW10[f] * POW10_NEGATIVE[e]), both as single expressions.
+ // Splitting the multiplies or reordering changes rounding.
+ double[] testValues = {0.123456789, 1.23456789, 12.3456789, 123.456789, 1234.56789};
+ for (double value : testValues) {
+ for (int e = 0; e <= 10; e++) {
+ for (int f = 0; f <= e; f++) {
+ if (!AlpEncoderDecoder.isDoubleException(value, e, f)) {
+ long encoded = AlpEncoderDecoder.encodeDouble(value, e, f);
+ double decoded = AlpEncoderDecoder.decodeDouble(encoded, e, f);
+ assertEquals(
+ "Roundtrip failed for " + value + " (e=" + e + ", f=" + f + ")",
+ Double.doubleToRawLongBits(value),
+ Double.doubleToRawLongBits(decoded));
+ }
+ }
+ }
+ }
+ }
+
+ // ========== Bit Width Tests ==========
+
+ @Test
+ public void testBitWidthForLong() {
+ assertEquals(0, AlpEncoderDecoder.bitWidthForLong(0L));
+ assertEquals(1, AlpEncoderDecoder.bitWidthForLong(1L));
+ assertEquals(2, AlpEncoderDecoder.bitWidthForLong(2L));
+ assertEquals(2, AlpEncoderDecoder.bitWidthForLong(3L));
+ assertEquals(3, AlpEncoderDecoder.bitWidthForLong(4L));
+ assertEquals(8, AlpEncoderDecoder.bitWidthForLong(255L));
+ assertEquals(9, AlpEncoderDecoder.bitWidthForLong(256L));
+ assertEquals(16, AlpEncoderDecoder.bitWidthForLong(65535L));
+ assertEquals(31, AlpEncoderDecoder.bitWidthForLong((long) Integer.MAX_VALUE));
+ assertEquals(63, AlpEncoderDecoder.bitWidthForLong(Long.MAX_VALUE));
+ }
+
+ // ========== Best Parameters Tests ==========
+
+ @Test
+ public void testFindBestFloatParams() {
+ float[] values = {1.23f, 4.56f, 7.89f, 10.11f, 12.13f};
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length);
+
+ assertNotNull(params);
+ assertTrue(params.exponent >= 0 && params.exponent <= AlpConstants.FLOAT_MAX_EXPONENT);
+ assertTrue(params.factor >= 0 && params.factor <= params.exponent);
+
+ for (float v : values) {
+ if (!AlpEncoderDecoder.isFloatException(v, params.exponent, params.factor)) {
+ int encoded = AlpEncoderDecoder.encodeFloat(v, params.exponent, params.factor);
+ float decoded = AlpEncoderDecoder.decodeFloat(encoded, params.exponent, params.factor);
+ assertEquals(Float.floatToRawIntBits(v), Float.floatToRawIntBits(decoded));
+ }
+ }
+ }
+
+ @Test
+ public void testFindBestFloatParamsWithAllExceptions() {
+ float[] values = {Float.NaN, Float.NaN, Float.NaN};
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length);
+
+ assertNotNull(params);
+ assertEquals(values.length, params.numExceptions);
+ }
+
+ @Test
+ public void testFindBestDoubleParams() {
+ double[] values = {1.23, 4.56, 7.89, 10.11, 12.13};
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestDoubleParams(values, 0, values.length);
+
+ assertNotNull(params);
+ assertTrue(params.exponent >= 0 && params.exponent <= AlpConstants.DOUBLE_MAX_EXPONENT);
+ assertTrue(params.factor >= 0 && params.factor <= params.exponent);
+
+ for (double v : values) {
+ if (!AlpEncoderDecoder.isDoubleException(v, params.exponent, params.factor)) {
+ long encoded = AlpEncoderDecoder.encodeDouble(v, params.exponent, params.factor);
+ double decoded = AlpEncoderDecoder.decodeDouble(encoded, params.exponent, params.factor);
+ assertEquals(Double.doubleToRawLongBits(v), Double.doubleToRawLongBits(decoded));
+ }
+ }
+ }
+
+ @Test
+ public void testFindBestDoubleParamsWithAllExceptions() {
+ double[] values = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY};
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestDoubleParams(values, 0, values.length);
+
+ assertNotNull(params);
+ assertEquals(values.length, params.numExceptions);
+ }
+
+ @Test
+ public void testFindBestParamsWithOffset() {
+ float[] values = {Float.NaN, Float.NaN, 1.23f, 4.56f, 7.89f, Float.NaN};
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 2, 3);
+
+ assertNotNull(params);
+ assertEquals(0, params.numExceptions);
+ }
+
+ // ========== Preset-Based Parameter Search Tests ==========
+
+ @Test
+ public void testFindBestFloatParamsWithPresets() {
+ float[] values = {1.23f, 4.56f, 7.89f, 10.11f, 12.13f};
+ int[][] presets = {{2, 0}, {3, 0}, {4, 1}};
+ AlpEncoderDecoder.EncodingParams params =
+ AlpEncoderDecoder.findBestFloatParamsWithPresets(values, 0, values.length, presets);
+
+ assertNotNull(params);
+ // Should select one of the preset combinations
+ boolean foundMatch = false;
+ for (int[] preset : presets) {
+ if (params.exponent == preset[0] && params.factor == preset[1]) {
+ foundMatch = true;
+ break;
+ }
+ }
+ assertTrue("Result should be one of the preset combinations", foundMatch);
+ }
+
+ @Test
+ public void testFindBestDoubleParamsWithPresets() {
+ double[] values = {1.23, 4.56, 7.89, 10.11, 12.13};
+ int[][] presets = {{2, 0}, {3, 0}, {4, 1}};
+ AlpEncoderDecoder.EncodingParams params =
+ AlpEncoderDecoder.findBestDoubleParamsWithPresets(values, 0, values.length, presets);
+
+ assertNotNull(params);
+ boolean foundMatch = false;
+ for (int[] preset : presets) {
+ if (params.exponent == preset[0] && params.factor == preset[1]) {
+ foundMatch = true;
+ break;
+ }
+ }
+ assertTrue("Result should be one of the preset combinations", foundMatch);
+ }
+
+ @Test
+ public void testPresetsProduceSameResultAsFullSearch() {
+ float[] values = {1.23f, 4.56f, 7.89f};
+ AlpEncoderDecoder.EncodingParams fullResult = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length);
+
+ // Include the best params in presets
+ int[][] presets = {{fullResult.exponent, fullResult.factor}, {0, 0}, {1, 0}};
+ AlpEncoderDecoder.EncodingParams presetResult =
+ AlpEncoderDecoder.findBestFloatParamsWithPresets(values, 0, values.length, presets);
+
+ assertTrue(
+ "Preset result should be at least as good as full search",
+ presetResult.numExceptions <= fullResult.numExceptions);
+ }
+
+ @Test
+ public void findBestFloatParamsMixedSignUsesSignedDeltaRange() {
+ // Mixed-sign, non-integer values are losslessly encodable at some (e, f). A buggy estimator
+ // that computes the FOR range with unsigned subtraction inflates every mixed-sign combo to a
+ // 64-bit width, making it prefer combos that turn values into exceptions. With the correct
+ // signed range (max - min), the estimator finds the lossless encoding with zero exceptions.
+ float[] values = {-3.14f, 2.71f, 100.5f, -50.25f};
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length);
+ assertEquals(0, params.numExceptions);
+ }
+
+ @Test
+ public void findBestFloatParamsWithPresetsMixedSignUsesSignedDeltaRange() {
+ // Restricting to two presets removes the equivalent-scaling ties, so the chosen exponent is
+ // deterministic: e=0,f=0 has the smallest signed FOR span and wins. A buggy unsigned estimator
+ // would rate both presets at 64 bits, tie, and pick the higher exponent (e=1) instead.
+ float[] values = {-1.0f, 1.0f};
+ int[][] presets = {{0, 0}, {1, 0}}; // e=0,f=0 has the smallest signed FOR span
+ AlpEncoderDecoder.EncodingParams params =
+ AlpEncoderDecoder.findBestFloatParamsWithPresets(values, 0, values.length, presets);
+ assertEquals(0, params.exponent);
+ assertEquals(0, params.factor);
+ assertEquals(0, params.numExceptions);
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpExceptionCountTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpExceptionCountTest.java
new file mode 100644
index 0000000000..592766d2f1
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpExceptionCountTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.column.values.alp;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Test;
+
+/**
+ * Measures exception counts and compression ratios using the real Spotify and Arade datasets from
+ * the parquet-testing repository (Pratik's alpFloatingPointDataset branch).
+ *
+ * These are the same datasets used in the meeting discussion to compare Java vs C++ ALP
+ * exception counts. Run from the project root with the alp-test-data/ directory present:
+ *
+ *
+ * ./mvnw test -pl parquet-column -Dtest=AlpExceptionCountTest
+ *
+ *
+ * To use a different data directory:
+ *
+ *
+ * ./mvnw test -pl parquet-column -Dtest=AlpExceptionCountTest \
+ * -DALP_TEST_DATA_DIR=/path/to/alp-test-data
+ *
+ */
+public class AlpExceptionCountTest {
+
+ private static final int VECTOR_SIZE = AlpConstants.DEFAULT_VECTOR_SIZE;
+
+ private File getDataDir() {
+ String dir = System.getProperty("ALP_TEST_DATA_DIR");
+ if (dir == null) dir = System.getenv("ALP_TEST_DATA_DIR");
+ if (dir != null && new File(dir).isDirectory()) return new File(dir);
+ // Default: alp-test-data/ at project root (three levels up from module target)
+ File candidate = new File(System.getProperty("user.dir")).getParentFile();
+ if (candidate != null) {
+ File d = new File(candidate, "alp-test-data");
+ if (d.isDirectory()) return d;
+ }
+ // Also try relative to cwd
+ File d2 = new File(System.getProperty("user.dir"), "alp-test-data");
+ return d2.isDirectory() ? d2 : null;
+ }
+
+ /** Reads a CSV file into per-column double arrays. Handles both ',' and '|' delimiters. */
+ private List readCsvColumns(File file) throws IOException {
+ List> cols = new ArrayList<>();
+ String delim = null;
+ try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+ String line;
+ boolean header = true;
+ while ((line = br.readLine()) != null) {
+ if (delim == null) delim = line.contains("|") ? "\\|" : ",";
+ String[] parts = line.split(delim);
+ if (header) {
+ for (String ignored : parts) cols.add(new ArrayList<>());
+ header = false;
+ continue;
+ }
+ for (int i = 0; i < parts.length && i < cols.size(); i++) {
+ try {
+ cols.get(i).add(Double.parseDouble(parts[i].trim()));
+ } catch (NumberFormatException e) {
+ cols.get(i).add(0.0);
+ }
+ }
+ }
+ }
+ List result = new ArrayList<>();
+ for (List col : cols) {
+ double[] arr = new double[col.size()];
+ for (int i = 0; i < arr.length; i++) arr[i] = col.get(i);
+ result.add(arr);
+ }
+ return result;
+ }
+
+ /** Reads the header row of a CSV file. */
+ private String[] readHeader(File file) throws IOException {
+ try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+ String line = br.readLine();
+ if (line == null) return new String[0];
+ String delim = line.contains("|") ? "\\|" : ",";
+ return line.split(delim);
+ }
+ }
+
+ /** Counts total exceptions across all vectors for a double column. */
+ private int countExceptions(double[] values) {
+ int totalExceptions = 0;
+ for (int offset = 0; offset < values.length; offset += VECTOR_SIZE) {
+ int len = Math.min(VECTOR_SIZE, values.length - offset);
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestDoubleParams(values, offset, len);
+ totalExceptions += params.numExceptions;
+ }
+ return totalExceptions;
+ }
+
+ private void reportDataset(String label, File file) throws IOException {
+ List columns = readCsvColumns(file);
+ String[] headers = readHeader(file);
+ int rows = columns.isEmpty() ? 0 : columns.get(0).length;
+ int numVectors = (int) Math.ceil((double) rows / VECTOR_SIZE);
+
+ System.out.printf("%n=== %s (%d rows, %d cols, %d vectors/col) ===%n", label, rows, columns.size(), numVectors);
+ System.out.printf(" %-20s %6s %6s %7s%n", "column", "rows", "exc", "exc%");
+ System.out.printf(" %-20s %6s %6s %7s%n", "------", "----", "---", "----");
+
+ int totalExc = 0;
+ int totalRows = 0;
+ for (int i = 0; i < columns.size(); i++) {
+ double[] col = columns.get(i);
+ int exc = countExceptions(col);
+ totalExc += exc;
+ totalRows += col.length;
+ String name = (i < headers.length) ? headers[i].trim() : "col" + i;
+ System.out.printf(" %-20s %6d %6d %6.2f%%%n", name, col.length, exc, 100.0 * exc / col.length);
+ }
+ System.out.printf(" %-20s %6d %6d %6.2f%%%n", "TOTAL", totalRows, totalExc, 100.0 * totalExc / totalRows);
+ }
+
+ @Test
+ public void testSpotifyExceptionCounts() throws IOException {
+ File dir = getDataDir();
+ assumeTrue("alp-test-data/ not found. Run from project root or set ALP_TEST_DATA_DIR", dir != null);
+ File file = new File(dir, "floatingpoint_spotify1.csv");
+ assumeTrue("floatingpoint_spotify1.csv not found in " + dir, file.exists());
+
+ reportDataset("Spotify", file);
+
+ // Basic sanity: Spotify data is well-behaved floats, expect < 5% exceptions overall
+ List columns = readCsvColumns(file);
+ int totalExc = 0, totalRows = 0;
+ for (double[] col : columns) {
+ totalExc += countExceptions(col);
+ totalRows += col.length;
+ }
+ double excRate = 100.0 * totalExc / totalRows;
+ System.out.printf("%nSpotify overall exception rate: %.2f%%%n", excRate);
+ assertTrue("Exception rate should be < 10% for Spotify data, got: " + excRate, excRate < 10.0);
+ }
+
+ @Test
+ public void testAradeExceptionCounts() throws IOException {
+ File dir = getDataDir();
+ assumeTrue("alp-test-data/ not found. Run from project root or set ALP_TEST_DATA_DIR", dir != null);
+ File file = new File(dir, "floatingpoint_arade.csv");
+ assumeTrue("floatingpoint_arade.csv not found in " + dir, file.exists());
+
+ reportDataset("Arade", file);
+
+ List columns = readCsvColumns(file);
+ int totalExc = 0, totalRows = 0;
+ for (double[] col : columns) {
+ totalExc += countExceptions(col);
+ totalRows += col.length;
+ }
+ double excRate = 100.0 * totalExc / totalRows;
+ System.out.printf("%nArade overall exception rate: %.2f%%%n", excRate);
+ assertTrue("Exception rate should be < 10% for Arade data, got: " + excRate, excRate < 10.0);
+ }
+
+ @Test
+ public void testAllDatasetsExceptionCounts() throws IOException {
+ File dir = getDataDir();
+ assumeTrue("alp-test-data/ not found. Run from project root or set ALP_TEST_DATA_DIR", dir != null);
+
+ File[] csvFiles = dir.listFiles((d, name) -> name.startsWith("floatingpoint_") && name.endsWith(".csv"));
+ assumeTrue("No floatingpoint_*.csv files found in " + dir, csvFiles != null && csvFiles.length > 0);
+
+ System.out.printf("%n=== All Datasets Summary ===%n");
+ System.out.printf(" %-30s %6s %6s %7s%n", "dataset", "rows", "exc", "exc%");
+ System.out.printf(" %-30s %6s %6s %7s%n", "-------", "----", "---", "----");
+
+ for (File file : csvFiles) {
+ List columns = readCsvColumns(file);
+ int totalExc = 0, totalRows = 0;
+ for (double[] col : columns) {
+ totalExc += countExceptions(col);
+ totalRows += col.length;
+ }
+ String name = file.getName().replace("floatingpoint_", "").replace(".csv", "");
+ System.out.printf(" %-30s %6d %6d %6.2f%%%n", name, totalRows, totalExc, 100.0 * totalExc / totalRows);
+ }
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpValuesEndToEndTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpValuesEndToEndTest.java
new file mode 100644
index 0000000000..793ac76a16
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpValuesEndToEndTest.java
@@ -0,0 +1,2202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Random;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.junit.Test;
+
+/**
+ * End-to-end tests for ALP encoding and decoding pipeline.
+ */
+public class AlpValuesEndToEndTest {
+
+ private static final int DEFAULT_VECTOR_SIZE = AlpConstants.DEFAULT_VECTOR_SIZE;
+
+ // ========== Helper Methods ==========
+
+ /**
+ * Round-trip with STRICT raw-bit comparison for every value, including NaN payloads (the regular
+ * roundTrip helpers only assert isNaN, which would hide a NaN-payload-preservation bug). ALP must
+ * be losslessly bit-exact for every possible IEEE-754 bit pattern.
+ */
+ private void roundTripDoubleStrict(double[] values) throws Exception {
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ int capacity = Math.max(512, values.length * 16);
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(
+ capacity, capacity, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ for (int i = 0; i < values.length; i++) {
+ long exp = Double.doubleToRawLongBits(values[i]);
+ long act = Double.doubleToRawLongBits(reader.readDouble());
+ assertEquals(
+ "Raw-bit mismatch at index " + i + " expectedBits=0x" + Long.toHexString(exp) + " actualBits=0x"
+ + Long.toHexString(act),
+ exp,
+ act);
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ private void roundTripFloatStrict(float[] values) throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ int capacity = Math.max(256, values.length * 8);
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(
+ capacity, capacity, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ for (int i = 0; i < values.length; i++) {
+ int exp = Float.floatToRawIntBits(values[i]);
+ int act = Float.floatToRawIntBits(reader.readFloat());
+ assertEquals(
+ "Raw-bit mismatch at index " + i + " expectedBits=0x" + Integer.toHexString(exp)
+ + " actualBits=0x" + Integer.toHexString(act),
+ exp,
+ act);
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Full-bit-space fuzz (lossless invariant) ==========
+
+ @Test
+ public void testDoubleFullBitSpaceFuzz() throws Exception {
+ // Sweep the entire double bit space: random raw longs -> double. Covers all subnormals, every
+ // NaN payload, +/-0, +/-Inf, extreme exponents, chaotic mixed magnitudes within a vector.
+ Random rng = new Random(0x9E3779B97F4A7C15L);
+ for (int v = 0; v < 300; v++) { // ~300k values
+ double[] values = new double[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = Double.longBitsToDouble(rng.nextLong());
+ }
+ roundTripDoubleStrict(values);
+ }
+ }
+
+ @Test
+ public void testFloatFullBitSpaceFuzz() throws Exception {
+ Random rng = new Random(0x9E3779B9L);
+ for (int v = 0; v < 300; v++) {
+ float[] values = new float[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = Float.intBitsToFloat(rng.nextInt());
+ }
+ roundTripFloatStrict(values);
+ }
+ }
+
+ @Test
+ public void testDoubleMixedFuzz() throws Exception {
+ // Mix ALP-friendly decimal-ish values with random raw bits, so FOR encoding AND the exception
+ // path are both exercised on chaotic data within the same vector.
+ Random rng = new Random(0xD1CE5EEDL);
+ for (int v = 0; v < 300; v++) {
+ double[] values = new double[1024];
+ for (int i = 0; i < values.length; i++) {
+ if (rng.nextInt(4) == 0) {
+ values[i] = Double.longBitsToDouble(rng.nextLong()); // ~25% chaotic (mostly exceptions)
+ } else {
+ values[i] = Math.round(rng.nextDouble() * 1_000_000.0) / 100.0; // 2-decimal, ALP-friendly
+ }
+ }
+ roundTripDoubleStrict(values);
+ }
+ }
+
+ @Test
+ public void testDoubleDistributionShiftWithinRowGroup() throws Exception {
+ // First vectors are clean 2-decimal data (the sampler builds its preset (e,f) cache from these),
+ // then later vectors switch to high-precision / very different magnitude that the cached presets
+ // fit poorly. Once presets are cached the writer only tries those, so ALP must still stay lossless
+ // (the exception path catches every preset mismatch).
+ Random rng = new Random(7);
+ int totalVectors = 40; // well past SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP so presets are in use
+ double[] values = new double[totalVectors * 1024];
+ for (int i = 0; i < values.length; i++) {
+ int vec = i / 1024;
+ if (vec < 12) {
+ values[i] = Math.round(rng.nextDouble() * 10000.0) / 100.0; // clean cents
+ } else {
+ values[i] = rng.nextDouble() * 1e12 + rng.nextDouble(); // high-precision, big magnitude
+ }
+ }
+ roundTripDoubleStrict(values);
+ }
+
+ // ========== Reader robustness against malformed input ==========
+
+ private void readAllDoubles(byte[] bytes, int valueCount) throws Exception {
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(valueCount, ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)));
+ for (int i = 0; i < valueCount; i++) {
+ reader.readDouble();
+ }
+ }
+
+ @Test(timeout = 30000)
+ public void testReaderRejectsCorruptInputCleanly() throws Exception {
+ // Build a valid ALP double page, then feed corrupted variants. The reader must fail cleanly
+ // (a catchable exception) and never hang, OOM, or read out of bounds silently.
+ double[] values = new double[2048];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = (i % 100) / 100.0;
+ }
+ AlpValuesWriter.DoubleAlpValuesWriter writer = new AlpValuesWriter.DoubleAlpValuesWriter(
+ 65536, 65536, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+ byte[] valid = writer.getBytes().toByteArray();
+ writer.reset();
+ writer.close();
+
+ // Sanity: the valid page reads fine.
+ readAllDoubles(valid, values.length);
+
+ // Corruptions that must each fail cleanly (never crash/hang/OOB):
+ java.util.List corrupt = new java.util.ArrayList<>();
+ corrupt.add(java.util.Arrays.copyOf(valid, valid.length / 2)); // truncated to half
+ corrupt.add(java.util.Arrays.copyOf(valid, 3)); // truncated to a stub
+ corrupt.add(new byte[0]); // empty
+ for (int pos : new int[] {0, 1, 2, 5, 7, 11, 20, valid.length - 1}) {
+ byte[] c = valid.clone();
+ c[pos] = (byte) ~c[pos]; // flip a header/body byte
+ corrupt.add(c);
+ }
+ for (byte[] c : corrupt) {
+ try {
+ readAllDoubles(c, values.length);
+ // Some single-byte flips may still decode to (wrong but in-bounds) values without throwing;
+ // that is acceptable here. What matters is no crash/hang/OOB, which we reached this line.
+ } catch (OutOfMemoryError oom) {
+ fail("Malformed input caused an OutOfMemoryError (allocation bomb) - a corrupt size/count "
+ + "must not drive an unbounded allocation");
+ } catch (Throwable t) {
+ // Any ordinary catchable exception (EOFException, ParquetDecodingException, IndexOutOfBounds,
+ // BufferUnderflow, NegativeArraySize, ...) is a clean failure: no JVM crash, no OOB, no hang.
+ }
+ }
+
+ // Claiming far more elements than the data supports must be rejected without an OOM allocation.
+ try {
+ readAllDoubles(valid, Integer.MAX_VALUE / 2);
+ fail("Expected a decoding failure for an absurd value count");
+ } catch (OutOfMemoryError oom) {
+ fail("Absurd value count drove an OutOfMemoryError instead of a clean rejection");
+ } catch (Throwable expected) {
+ // clean rejection
+ }
+ }
+
+ // ========== Large scale + allocator leak ==========
+
+ /** Wraps an allocator and tracks outstanding (allocated but not released) buffers. */
+ private static final class CountingAllocator implements org.apache.parquet.bytes.ByteBufferAllocator {
+ private final org.apache.parquet.bytes.ByteBufferAllocator delegate;
+ final java.util.concurrent.atomic.AtomicInteger outstanding = new java.util.concurrent.atomic.AtomicInteger();
+ final java.util.concurrent.atomic.AtomicInteger peakOutstanding =
+ new java.util.concurrent.atomic.AtomicInteger();
+
+ CountingAllocator(org.apache.parquet.bytes.ByteBufferAllocator delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public ByteBuffer allocate(int size) {
+ int cur = outstanding.incrementAndGet();
+ peakOutstanding.getAndAccumulate(cur, Math::max);
+ return delegate.allocate(size);
+ }
+
+ @Override
+ public void release(ByteBuffer b) {
+ outstanding.decrementAndGet();
+ delegate.release(b);
+ }
+
+ @Override
+ public boolean isDirect() {
+ return delegate.isDirect();
+ }
+ }
+
+ @Test
+ public void testWriterReleasesAllAllocatorBuffersOverManyPages() throws Exception {
+ // Simulate many pages (write -> getBytes -> reset, repeated) then close. A per-page slab leak
+ // (the GH-3628-style unreleased-buffer bug) would leave outstanding allocations > 0 after close.
+ CountingAllocator alloc = new CountingAllocator(new DirectByteBufferAllocator());
+ AlpValuesWriter.DoubleAlpValuesWriter writer =
+ new AlpValuesWriter.DoubleAlpValuesWriter(1024, 1024, alloc, DEFAULT_VECTOR_SIZE);
+ Random rng = new Random(1);
+ try {
+ for (int page = 0; page < 200; page++) {
+ for (int i = 0; i < 5000; i++) {
+ writer.writeDouble(Math.round(rng.nextDouble() * 100000.0) / 100.0);
+ }
+ writer.getBytes();
+ writer.reset();
+ }
+ } finally {
+ writer.close();
+ }
+ assertTrue("Writer should have allocated buffers during the run", alloc.peakOutstanding.get() > 0);
+ assertEquals(
+ "Writer leaked allocator buffers: " + alloc.outstanding.get() + " outstanding after close",
+ 0,
+ alloc.outstanding.get());
+ }
+
+ @Test
+ public void testLargeScaleDoubleRoundTrip() throws Exception {
+ // ~2M values across many pages/vectors: verifies correctness at scale and that nothing OOMs.
+ Random rng = new Random(2);
+ int perPage = 100_000;
+ int pages = 20;
+ for (int p = 0; p < pages; p++) {
+ double[] values = new double[perPage];
+ for (int i = 0; i < perPage; i++) {
+ values[i] = Math.round(rng.nextDouble() * 1e7) / 100.0;
+ }
+ roundTripDoubleStrict(values);
+ }
+ }
+
+ private void roundTripFloat(float[] values) throws Exception {
+ roundTripFloat(values, DEFAULT_VECTOR_SIZE);
+ }
+
+ private void roundTripFloat(float[] values, int vectorSize) throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ int capacity = Math.max(256, values.length * 8);
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(
+ capacity, capacity, new DirectByteBufferAllocator(), vectorSize);
+
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ float expected = values[i];
+ float actual = reader.readFloat();
+
+ if (Float.isNaN(expected)) {
+ assertTrue("Expected NaN at index " + i, Float.isNaN(actual));
+ } else {
+ assertEquals(
+ "Value mismatch at index " + i + " for " + expected,
+ Float.floatToRawIntBits(expected),
+ Float.floatToRawIntBits(actual));
+ }
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ private void roundTripDouble(double[] values) throws Exception {
+ roundTripDouble(values, DEFAULT_VECTOR_SIZE);
+ }
+
+ private void roundTripDouble(double[] values, int vectorSize) throws Exception {
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ int capacity = Math.max(512, values.length * 16);
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(
+ capacity, capacity, new DirectByteBufferAllocator(), vectorSize);
+
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ double expected = values[i];
+ double actual = reader.readDouble();
+
+ if (Double.isNaN(expected)) {
+ assertTrue("Expected NaN at index " + i, Double.isNaN(actual));
+ } else {
+ assertEquals(
+ "Value mismatch at index " + i + " for " + expected,
+ Double.doubleToRawLongBits(expected),
+ Double.doubleToRawLongBits(actual));
+ }
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Float Pipeline Tests ==========
+
+ @Test
+ public void testFloatSingleValue() throws Exception {
+ roundTripFloat(new float[] {1.23f});
+ }
+
+ @Test
+ public void testFloatSmallBatch() throws Exception {
+ roundTripFloat(new float[] {0.0f, 1.0f, -1.0f, 3.14f, 100.5f, 0.001f, 1234567.0f});
+ }
+
+ @Test
+ public void testFloatRandomData() throws Exception {
+ Random rand = new Random(42);
+ float[] values = new float[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextFloat() * 10000.0f - 5000.0f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatWithExceptions() throws Exception {
+ roundTripFloat(new float[] {
+ 1.0f, Float.NaN, 2.0f, Float.POSITIVE_INFINITY, 3.0f, Float.NEGATIVE_INFINITY, 4.0f, -0.0f, 5.0f
+ });
+ }
+
+ @Test
+ public void testFloatAllExceptions() throws Exception {
+ roundTripFloat(new float[] {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, -0.0f});
+ }
+
+ @Test
+ public void testFloatMultipleVectors() throws Exception {
+ Random rand = new Random(42);
+ float[] values = new float[3000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextFloat() * 1000.0f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatZeroValues() throws Exception {
+ roundTripFloat(new float[] {0.0f, 0.0f, 0.0f, 0.0f});
+ }
+
+ @Test
+ public void testFloatWideForRange() throws Exception {
+ // Integer-valued floats spanning the widest signed range float holds losslessly (< 2^23), so the
+ // frame-of-reference minimum is deeply negative and the FOR delta needs a high bit width. A buggy
+ // unsigned (max - min) range estimate would misjudge this; the signed range must round-trip cleanly.
+ float[] values = new float[2048];
+ float lo = -8.3e6f, hi = 8.3e6f;
+ for (int i = 0; i < values.length; i++) {
+ values[i] = (i % 2 == 0) ? lo + i : hi - i;
+ }
+ values[0] = lo; // frame minimum near -2^23
+ values[1] = hi; // frame maximum near +2^23
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testDoubleWideForRange() throws Exception {
+ // Integer-valued doubles spanning a wide signed range (< 2^52) so the frame-of-reference minimum
+ // is deeply negative and the FOR delta needs a near-53-bit width, all with zero exceptions.
+ double[] values = new double[2048];
+ double lo = -4.5e15, hi = 4.5e15;
+ for (int i = 0; i < values.length; i++) {
+ values[i] = (i % 2 == 0) ? lo + i : hi - i;
+ }
+ values[0] = lo; // frame minimum near -2^52
+ values[1] = hi; // frame maximum near +2^52
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoubleExtremeForBitWidth() throws Exception {
+ // Extreme values near the ALP double encoding limit (~2^63), all exact integer-valued doubles so
+ // they encode losslessly with no exceptions, but the frame-of-reference delta spans the full range
+ // and overflows the signed subtraction, forcing 64-bit FOR packing. Verified with the encoder:
+ // scale of 1 (e == f), 0 exceptions, FOR bit width 64. Exercises 64-bit packing and the reader's
+ // modular reconstruction, which cross-language interop must match.
+ double[] values = new double[2048];
+ long lo = -9_000_000_000_000_000_000L, hi = 9_000_000_000_000_000_000L;
+ for (int i = 0; i < values.length; i++) {
+ long off = (long) (i % 256) * (1L << 20);
+ values[i] = (double) ((i % 2 == 0) ? (lo + off) : (hi - off));
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoubleExtremeFor63BitWidth() throws Exception {
+ // Exact integer-valued doubles near +/- 2^61, so the frame-of-reference delta spans ~2^62.5 and
+ // needs 63 bits WITHOUT overflowing the signed max - min subtraction (the non-overflow high-bit
+ // counterpart to the 64-bit case above). Verified with the encoder: 0 exceptions, FOR bit width 63.
+ double[] values = new double[2048];
+ long lo = -3_000_000_000_000_000_000L, hi = 3_000_000_000_000_000_000L;
+ for (int i = 0; i < values.length; i++) {
+ long off = (long) (i % 256) * (1L << 20);
+ values[i] = (double) ((i % 2 == 0) ? (lo + off) : (hi - off));
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatExtremeForBitWidth() throws Exception {
+ // Float analogue near the ~2^31 float encoding limit: exact integer-valued floats whose encoded
+ // ints span ~2^32, forcing the full 32-bit FOR width. Verified: e=0, f=0, 0 exceptions, bit width 32.
+ float[] values = new float[2048];
+ int lo = -2_100_000_000, hi = 2_100_000_000;
+ for (int i = 0; i < values.length; i++) {
+ int off = (i % 256) * (1 << 8);
+ values[i] = (float) ((i % 2 == 0) ? (lo + off) : (hi - off));
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatAllNullPage() throws Exception {
+ // Every row on the page is null, so no value reaches the writer and num_elements is 0.
+ // numVectors then works out to 0 and the reader ends up calling slice(0) on empty offset and
+ // vector buffers. Initialization must succeed even though the page carries a positive row count.
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(
+ 256, 256, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ // no writeFloat calls: all rows are null
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(100, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleAllNullPage() throws Exception {
+ // Double analogue of testFloatAllNullPage: an all-null page produces num_elements 0.
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(
+ 512, 512, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ // no writeDouble calls: all rows are null
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(100, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatPartialNullPage() throws Exception {
+ // Some rows are null, so fewer values reach the writer than the page's row count: num_elements
+ // is less than valueCount. The reader is initialized with the larger page count and must serve
+ // exactly the num_elements encoded values back.
+ int numValues = 700;
+ int pageRowCount = 1000; // 300 nulls
+ float[] values = new float[numValues];
+ for (int i = 0; i < numValues; i++) {
+ values[i] = (i % 100) / 100.0f + 1.0f;
+ }
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(
+ 16384, 16384, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(pageRowCount, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ for (int i = 0; i < numValues; i++) {
+ assertEquals(
+ "Value mismatch at index " + i,
+ Float.floatToRawIntBits(values[i]),
+ Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDoublePartialNullPage() throws Exception {
+ // Double analogue of testFloatPartialNullPage: num_elements less than valueCount because some
+ // rows are null.
+ int numValues = 700;
+ int pageRowCount = 1000; // 300 nulls
+ double[] values = new double[numValues];
+ for (int i = 0; i < numValues; i++) {
+ values[i] = (i % 100) / 100.0 + 1.0;
+ }
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(
+ 32768, 32768, new DirectByteBufferAllocator(), DEFAULT_VECTOR_SIZE);
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(pageRowCount, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ for (int i = 0; i < numValues; i++) {
+ assertEquals(
+ "Value mismatch at index " + i,
+ Double.doubleToRawLongBits(values[i]),
+ Double.doubleToRawLongBits(reader.readDouble()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatIdenticalValues() throws Exception {
+ float[] values = new float[100];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 3.14159f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== Float: Vector Size / Boundary Tests ==========
+
+ @Test
+ public void testFloatExactOneVector() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatExactTwoVectors() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE * 2];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatExactThreeVectors() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE * 3];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatPartialLastVectorPlusOne() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE + 1];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatPartialLastVectorMinusOne() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE * 2 - 1];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testFloatCustomVectorSize512() throws Exception {
+ float[] values = new float[512 + 100]; // partial second vector
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.1f;
+ }
+ roundTripFloat(values, 512);
+ }
+
+ @Test
+ public void testFloatCustomVectorSize2048() throws Exception {
+ float[] values = new float[2048 * 2 + 500]; // partial third vector
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.1f;
+ }
+ roundTripFloat(values, 2048);
+ }
+
+ @Test
+ public void testFloatCustomVectorSizeSmall() throws Exception {
+ // Minimum vector size = 8 (2^3)
+ float[] values = new float[25]; // 3 full vectors + 1 partial
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 1.5f;
+ }
+ roundTripFloat(values, 8);
+ }
+
+ // ========== Float: Large Page ==========
+
+ @Test
+ public void testFloatLargePage() throws Exception {
+ Random rand = new Random(42);
+ float[] values = new float[15000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextFloat() * 100000.0f - 50000.0f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== Float: Monetary Data ==========
+
+ @Test
+ public void testFloatMonetaryData() throws Exception {
+ float[] values = {
+ 19.99f, 29.95f, 9.99f, 49.99f, 99.95f, 14.50f, 7.99f, 24.99f, 39.95f, 59.99f, 119.99f, 4.99f, 0.99f, 1.50f,
+ 2.99f, 3.49f
+ };
+ roundTripFloat(values);
+ }
+
+ // ========== Float: All Identical (bit_width = 0) ==========
+
+ @Test
+ public void testFloatAllIdenticalBitWidthZero() throws Exception {
+ float[] values = new float[2000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 42.0f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== Float: Mixed Exceptions at Various Positions ==========
+
+ @Test
+ public void testFloatExceptionsAtBoundaries() throws Exception {
+ // Exceptions at vector start and end
+ float[] values = new float[20];
+ values[0] = Float.NaN;
+ for (int i = 1; i < values.length - 1; i++) {
+ values[i] = i * 1.1f;
+ }
+ values[values.length - 1] = Float.POSITIVE_INFINITY;
+ roundTripFloat(values);
+ }
+
+ // ========== Float: Skip Tests ==========
+
+ @Test
+ public void testFloatSkip() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+
+ float[] values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f};
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertEquals(Float.floatToRawIntBits(1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ reader.skip(3);
+ assertEquals(Float.floatToRawIntBits(5.0f), Float.floatToRawIntBits(reader.readFloat()));
+ reader.skip(2);
+ assertEquals(Float.floatToRawIntBits(8.0f), Float.floatToRawIntBits(reader.readFloat()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testFloatSkipAcrossVectorBoundaries() throws Exception {
+ int vectorSize = 8; // small vector for boundary testing
+ int totalValues = vectorSize * 3 + 4; // 3 full vectors + partial
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(
+ totalValues * 8, totalValues * 8, new DirectByteBufferAllocator(), vectorSize);
+
+ float[] values = new float[totalValues];
+ for (int i = 0; i < totalValues; i++) {
+ values[i] = (i + 1) * 1.0f;
+ writer.writeFloat(values[i]);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(totalValues, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ // Read first value from vector 0
+ assertEquals(Float.floatToRawIntBits(1.0f), Float.floatToRawIntBits(reader.readFloat()));
+
+ // Skip past vector 0 and vector 1, into vector 2
+ reader.skip(vectorSize * 2 - 1); // skip to index vectorSize*2
+
+ // Read first value of vector 2
+ float expected = (vectorSize * 2 + 1) * 1.0f;
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(reader.readFloat()));
+
+ // Skip into the partial last vector
+ reader.skip(vectorSize - 1); // skip to index vectorSize*3
+
+ expected = (vectorSize * 3 + 1) * 1.0f;
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(reader.readFloat()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Float: Writer Reset ==========
+
+ @Test
+ public void testFloatWriterReset() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+ writer.reset();
+ assertEquals(0, writer.getBufferedSize());
+
+ float[] values = {3.0f, 4.0f, 5.0f};
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (float expected : values) {
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Double Pipeline Tests ==========
+
+ @Test
+ public void testDoubleSingleValue() throws Exception {
+ roundTripDouble(new double[] {1.23456789});
+ }
+
+ @Test
+ public void testDoubleSmallBatch() throws Exception {
+ roundTripDouble(new double[] {0.0, 1.0, -1.0, 3.14159265358979, 100.5, 0.001, 12345678901234.0});
+ }
+
+ @Test
+ public void testDoubleRandomData() throws Exception {
+ Random rand = new Random(42);
+ double[] values = new double[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextDouble() * 1000000.0 - 500000.0;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoubleWithExceptions() throws Exception {
+ roundTripDouble(new double[] {
+ 1.0, Double.NaN, 2.0, Double.POSITIVE_INFINITY, 3.0, Double.NEGATIVE_INFINITY, 4.0, -0.0, 5.0
+ });
+ }
+
+ @Test
+ public void testDoubleAllExceptions() throws Exception {
+ roundTripDouble(new double[] {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, -0.0});
+ }
+
+ @Test
+ public void testDoubleMultipleVectors() throws Exception {
+ Random rand = new Random(42);
+ double[] values = new double[3000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextDouble() * 100000.0;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoubleZeroValues() throws Exception {
+ roundTripDouble(new double[] {0.0, 0.0, 0.0, 0.0});
+ }
+
+ @Test
+ public void testDoubleIdenticalValues() throws Exception {
+ double[] values = new double[100];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 3.14159265358979;
+ }
+ roundTripDouble(values);
+ }
+
+ // ========== Double: Vector Size / Boundary Tests ==========
+
+ @Test
+ public void testDoubleExactOneVector() throws Exception {
+ double[] values = new double[DEFAULT_VECTOR_SIZE];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoubleExactTwoVectors() throws Exception {
+ double[] values = new double[DEFAULT_VECTOR_SIZE * 2];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoublePartialLastVectorPlusOne() throws Exception {
+ double[] values = new double[DEFAULT_VECTOR_SIZE + 1];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoublePartialLastVectorMinusOne() throws Exception {
+ double[] values = new double[DEFAULT_VECTOR_SIZE * 2 - 1];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.01;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testDoubleCustomVectorSize512() throws Exception {
+ double[] values = new double[512 + 100];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.1;
+ }
+ roundTripDouble(values, 512);
+ }
+
+ @Test
+ public void testDoubleCustomVectorSize2048() throws Exception {
+ double[] values = new double[2048 * 2 + 500];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 0.1;
+ }
+ roundTripDouble(values, 2048);
+ }
+
+ @Test
+ public void testDoubleCustomVectorSizeSmall() throws Exception {
+ double[] values = new double[25];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i * 1.5;
+ }
+ roundTripDouble(values, 8);
+ }
+
+ // ========== Double: Large Page ==========
+
+ @Test
+ public void testDoubleLargePage() throws Exception {
+ Random rand = new Random(42);
+ double[] values = new double[15000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextDouble() * 100000.0 - 50000.0;
+ }
+ roundTripDouble(values);
+ }
+
+ // ========== Double: Monetary Data ==========
+
+ @Test
+ public void testDoubleMonetaryData() throws Exception {
+ double[] values = {
+ 19.99, 29.95, 9.99, 49.99, 99.95, 14.50, 7.99, 24.99, 39.95, 59.99, 119.99, 4.99, 0.99, 1.50, 2.99, 3.49
+ };
+ roundTripDouble(values);
+ }
+
+ // ========== Double: All Identical (bit_width = 0) ==========
+
+ @Test
+ public void testDoubleAllIdenticalBitWidthZero() throws Exception {
+ double[] values = new double[2000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 42.0;
+ }
+ roundTripDouble(values);
+ }
+
+ // ========== Double: Mixed Exceptions at Boundaries ==========
+
+ @Test
+ public void testDoubleExceptionsAtBoundaries() throws Exception {
+ double[] values = new double[20];
+ values[0] = Double.NaN;
+ for (int i = 1; i < values.length - 1; i++) {
+ values[i] = i * 1.1;
+ }
+ values[values.length - 1] = Double.POSITIVE_INFINITY;
+ roundTripDouble(values);
+ }
+
+ // ========== Double: Skip Tests ==========
+
+ @Test
+ public void testDoubleSkip() throws Exception {
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(512, 512, new DirectByteBufferAllocator());
+
+ double[] values = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0};
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertEquals(Double.doubleToRawLongBits(1.0), Double.doubleToRawLongBits(reader.readDouble()));
+ reader.skip(3);
+ assertEquals(Double.doubleToRawLongBits(5.0), Double.doubleToRawLongBits(reader.readDouble()));
+ reader.skip(2);
+ assertEquals(Double.doubleToRawLongBits(8.0), Double.doubleToRawLongBits(reader.readDouble()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleSkipAcrossVectorBoundaries() throws Exception {
+ int vectorSize = 8;
+ int totalValues = vectorSize * 3 + 4;
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(
+ totalValues * 16, totalValues * 16, new DirectByteBufferAllocator(), vectorSize);
+
+ double[] values = new double[totalValues];
+ for (int i = 0; i < totalValues; i++) {
+ values[i] = (i + 1) * 1.0;
+ writer.writeDouble(values[i]);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(totalValues, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertEquals(Double.doubleToRawLongBits(1.0), Double.doubleToRawLongBits(reader.readDouble()));
+ reader.skip(vectorSize * 2 - 1);
+
+ double expected = (vectorSize * 2 + 1) * 1.0;
+ assertEquals(Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(reader.readDouble()));
+
+ reader.skip(vectorSize - 1);
+ expected = (vectorSize * 3 + 1) * 1.0;
+ assertEquals(Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(reader.readDouble()));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Double: Writer Reset ==========
+
+ @Test
+ public void testDoubleWriterReset() throws Exception {
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(512, 512, new DirectByteBufferAllocator());
+
+ writer.writeDouble(1.0);
+ writer.writeDouble(2.0);
+ writer.reset();
+ assertEquals(0, writer.getBufferedSize());
+
+ double[] values = {3.0, 4.0, 5.0};
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (double expected : values) {
+ assertEquals(Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(reader.readDouble()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Packed Size Verification ==========
+
+ @Test
+ public void testPackedSizeSpecFormula() throws Exception {
+ // Verify that the packed data size exactly matches ceil(n * bitWidth / 8)
+ // by checking total encoded bytes for partial vectors with various remainders.
+ // Vector size 8 means: 1 full group of 8 = no partial group,
+ // but with count not divisible by 8 we exercise partial groups.
+ for (int count : new int[] {1, 2, 3, 5, 7, 9, 11, 13, 15}) {
+ float[] values = new float[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = (i + 1) * 1.0f;
+ }
+ // Use smallest vector size so the partial vector IS the only vector
+ roundTripFloat(values, 1024);
+ }
+ }
+
+ @Test
+ public void testPartialVectorAllRemaindersMod8() throws Exception {
+ // Test every possible remainder (1..7) to exercise the partial group path.
+ // With vectorSize=16: 16 is 2 full groups of 8, no partial group.
+ // So use vectorSize=16 and write 16+R values where R in {1..7}
+ // to get a partial last vector of size R.
+ for (int remainder = 1; remainder <= 7; remainder++) {
+ int count = 16 + remainder;
+ float[] values = new float[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = (i + 1) * 0.5f;
+ }
+ roundTripFloat(values, 16);
+ }
+ // Also test for doubles
+ for (int remainder = 1; remainder <= 7; remainder++) {
+ int count = 16 + remainder;
+ double[] values = new double[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = (i + 1) * 0.5;
+ }
+ roundTripDouble(values, 16);
+ }
+ }
+
+ // ========== Negative Encoded Values / Wide FOR Range ==========
+
+ @Test
+ public void testFloatNegativeValues() throws Exception {
+ // Values that produce negative encoded integers, testing FOR with negative min
+ float[] values = new float[100];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = -50.0f + i * 1.0f; // range [-50, 49]
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testDoubleNegativeValues() throws Exception {
+ double[] values = new double[100];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = -50.0 + i * 1.0;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatWideRange() throws Exception {
+ // Mix of small and large positive values - exercises higher bit widths
+ float[] values = {0.01f, 0.02f, 1000.0f, 2000.0f, 0.05f, 50000.0f, 0.99f, 99999.0f};
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testDoubleWideRange() throws Exception {
+ double[] values = {0.01, 0.02, 100000.0, 200000.0, 0.05, 5000000.0, 0.99, 99999999.0};
+ roundTripDouble(values);
+ }
+
+ // ========== Binary Format Verification ==========
+
+ @Test
+ public void testBinaryFormatExactBytes() throws Exception {
+ // Write a known small dataset with vectorSize=8, then verify the
+ // exact binary layout byte-by-byte
+ int vectorSize = 8;
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), vectorSize);
+
+ // Write 10 identical values: 1.0f
+ // With e=0, f=0: encoded=1, FOR min=1, all deltas=0, bitWidth=0
+ for (int i = 0; i < 10; i++) {
+ writer.writeFloat(1.0f);
+ }
+
+ BytesInput input = writer.getBytes();
+ byte[] bytes = input.toByteArray();
+ java.nio.ByteBuffer buf = java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ assertEquals("compression_mode", 0, buf.get() & 0xFF);
+ assertEquals("integer_encoding", 0, buf.get() & 0xFF);
+ assertEquals("log_vector_size", 3, buf.get() & 0xFF); // log2(8)=3
+ assertEquals("num_elements", 10, buf.getInt());
+
+ // Offset array (2 vectors * 4 bytes = 8 bytes)
+ int offset0 = buf.getInt(); // first vector offset
+ int offset1 = buf.getInt(); // second vector offset
+ assertEquals("offset0 = past offset array", 8, offset0);
+
+ // Vector 0: 8 identical values, bitWidth=0, no exceptions
+ // AlpInfo (4 bytes)
+ int v0Exp = buf.get() & 0xFF;
+ int v0Fac = buf.get() & 0xFF;
+ int v0Exc = buf.getShort() & 0xFFFF;
+ assertEquals("v0 exceptions", 0, v0Exc);
+
+ // ForInfo (5 bytes)
+ int v0For = buf.getInt();
+ int v0Bw = buf.get() & 0xFF;
+ assertEquals("v0 bitWidth", 0, v0Bw);
+ // No packed bytes (bitWidth=0)
+ // No exceptions
+
+ // Vector 0 should be exactly 4 + 5 = 9 bytes
+ assertEquals("offset1 should be offset0 + 9", offset0 + 9, offset1);
+
+ // Vector 1: 2 identical values, bitWidth=0, no exceptions
+ int v1Exp = buf.get() & 0xFF;
+ int v1Fac = buf.get() & 0xFF;
+ assertEquals("same exponent both vectors", v0Exp, v1Exp);
+ assertEquals("same factor both vectors", v0Fac, v1Fac);
+ int v1Exc = buf.getShort() & 0xFFFF;
+ assertEquals("v1 exceptions", 0, v1Exc);
+ int v1For = buf.getInt();
+ int v1Bw = buf.get() & 0xFF;
+ assertEquals("v1 bitWidth", 0, v1Bw);
+
+ // Should have consumed all bytes
+ assertEquals("all bytes consumed", bytes.length, buf.position());
+
+ // Verify round-trip
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(10, ByteBufferInputStream.wrap(input.toByteBuffer()));
+ for (int i = 0; i < 10; i++) {
+ assertEquals(Float.floatToRawIntBits(1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Exception-Heavy Vectors ==========
+
+ @Test
+ public void testFloatManyExceptionsInVector() throws Exception {
+ // Vector with more exceptions than normal values
+ float[] values = new float[10];
+ values[0] = 1.0f; // normal
+ for (int i = 1; i < 10; i++) {
+ values[i] = Float.NaN; // all exceptions except first
+ }
+ roundTripFloat(values, 16);
+ }
+
+ @Test
+ public void testDoubleManyExceptionsInVector() throws Exception {
+ double[] values = new double[10];
+ values[0] = 1.0;
+ for (int i = 1; i < 10; i++) {
+ values[i] = Double.NaN;
+ }
+ roundTripDouble(values, 16);
+ }
+
+ @Test
+ public void testFloatExceptionsAcrossMultipleVectors() throws Exception {
+ // Exceptions in different vectors
+ int vectorSize = 8;
+ float[] values = new float[vectorSize * 3];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = (i + 1) * 0.5f;
+ }
+ // Place exceptions at specific positions in different vectors
+ values[0] = Float.NaN; // vector 0, position 0
+ values[vectorSize - 1] = Float.POSITIVE_INFINITY; // vector 0, last position
+ values[vectorSize] = -0.0f; // vector 1, position 0
+ values[vectorSize * 2 + 3] = Float.NaN; // vector 2, position 3
+ roundTripFloat(values, vectorSize);
+ }
+
+ // ========== Read After Skip to End ==========
+
+ @Test(expected = ParquetDecodingException.class)
+ public void testFloatReadPastEnd() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(1, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ reader.readFloat(); // read the one value
+ reader.readFloat(); // should throw
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test(expected = ParquetDecodingException.class)
+ public void testFloatSkipPastEnd() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(2, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ reader.skip(3); // should throw - only 2 values
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Hand-Crafted Binary → Reader (independent of writer) ==========
+
+ /**
+ * Construct ALP-encoded bytes manually per the spec and verify the reader
+ * decodes them correctly. This catches symmetric bugs where writer and reader
+ * both implement the same wrong thing.
+ *
+ * Test vector: 3 float values {100.0f, 200.0f, 300.0f} with vectorSize=8.
+ * With exponent=0, factor=0: encoded = {100, 200, 300}.
+ * FOR min = 100, deltas = {0, 100, 200}, maxDelta = 200, bitWidth = 8.
+ * Packed: 3 values in 1 partial group of 8, padded with zeros.
+ * Packed size = ceil(3 * 8 / 8) = 3 bytes.
+ */
+ @Test
+ public void testReaderWithHandCraftedBytes() throws Exception {
+ // Manually build ALP page for {100.0f, 200.0f, 300.0f}, vectorSize=8
+ int vectorSize = 8;
+ int logVectorSize = 3;
+ int numElements = 3;
+ int numVectors = 1;
+
+ // Encoding: e=0, f=0 → multiplier=1.0
+ // encoded = {100, 200, 300}
+ // FOR min = 100
+ // deltas = {0, 100, 200}
+ // maxDelta = 200, bitWidth = 8
+ int exponent = 0;
+ int factor = 0;
+ int numExceptions = 0;
+ int frameOfReference = 100;
+ int bitWidth = 8;
+
+ // Bit-pack deltas using the same BytePacker the reader will use
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(8);
+ int[] deltas = {0, 100, 200, 0, 0, 0, 0, 0}; // padded to 8
+ byte[] packed = new byte[8]; // bitWidth bytes for one group of 8
+ packer.pack8Values(deltas, 0, packed, 0);
+ int packedSize = (3 * 8 + 7) / 8; // = 3 bytes for 3 values at 8 bits
+
+ // Build the page
+ int vectorDataSize = 4 + 5 + packedSize; // AlpInfo + ForInfo + packed
+ int offsetArraySize = numVectors * 4;
+
+ ByteBuffer page =
+ ByteBuffer.allocate(7 + offsetArraySize + vectorDataSize).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) logVectorSize);
+ page.putInt(numElements);
+
+ // Offset array (1 vector)
+ page.putInt(offsetArraySize); // offset₀ = past offset array
+
+ // Vector 0: AlpInfo (4 bytes)
+ page.put((byte) exponent);
+ page.put((byte) factor);
+ page.putShort((short) numExceptions);
+
+ // Vector 0: ForInfo (5 bytes)
+ page.putInt(frameOfReference);
+ page.put((byte) bitWidth);
+
+ // Vector 0: Packed data (3 bytes)
+ page.put(packed, 0, packedSize);
+
+ page.flip();
+
+ // Now read it back
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ assertEquals(Float.floatToRawIntBits(100.0f), Float.floatToRawIntBits(reader.readFloat()));
+ assertEquals(Float.floatToRawIntBits(200.0f), Float.floatToRawIntBits(reader.readFloat()));
+ assertEquals(Float.floatToRawIntBits(300.0f), Float.floatToRawIntBits(reader.readFloat()));
+ }
+
+ /**
+ * Hand-craft bytes with exceptions to verify reader handles them independently.
+ * Values: {1.0f, NaN, 3.0f} with vectorSize=8.
+ * e=0, f=0: encoded={1, placeholder=1, 3}, 1 exception at position 1.
+ * FOR min=1, deltas={0, 0, 2}, maxDelta=2, bitWidth=2.
+ */
+ @Test
+ public void testReaderWithHandCraftedExceptions() throws Exception {
+ int vectorSize = 8;
+ int logVectorSize = 3;
+ int numElements = 3;
+
+ int exponent = 0;
+ int factor = 0;
+ int numExceptions = 1;
+ int placeholder = 1; // first non-exception encoded value
+ int frameOfReference = 1; // min of {1, 1, 3}
+ int bitWidth = 2; // ceil(log2(2+1)) = 2
+
+ // deltas = {1-1, 1-1, 3-1} = {0, 0, 2}, padded to 8: {0,0,2,0,0,0,0,0}
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(2);
+ int[] deltas = {0, 0, 2, 0, 0, 0, 0, 0};
+ byte[] packed = new byte[2]; // bitWidth=2 bytes for a group of 8
+ packer.pack8Values(deltas, 0, packed, 0);
+ int packedSize = (3 * 2 + 7) / 8; // = 1 byte
+
+ // Vector data: AlpInfo(4) + ForInfo(5) + packed(1) + excPos(2) + excVal(4) = 16
+ int vectorDataSize = 4 + 5 + packedSize + 2 + 4;
+ int offsetArraySize = 1 * 4;
+
+ ByteBuffer page =
+ ByteBuffer.allocate(7 + offsetArraySize + vectorDataSize).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) logVectorSize);
+ page.putInt(numElements);
+
+ // Offset array
+ page.putInt(offsetArraySize);
+
+ // AlpInfo
+ page.put((byte) exponent);
+ page.put((byte) factor);
+ page.putShort((short) numExceptions);
+
+ // ForInfo
+ page.putInt(frameOfReference);
+ page.put((byte) bitWidth);
+
+ // Packed data
+ page.put(packed, 0, packedSize);
+
+ // Exception positions (uint16 LE)
+ page.putShort((short) 1); // position 1
+
+ // Exception values (float32 LE)
+ page.putFloat(Float.NaN);
+
+ page.flip();
+
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ assertEquals(Float.floatToRawIntBits(1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ float val1 = reader.readFloat();
+ assertTrue("Expected NaN at position 1", Float.isNaN(val1));
+ assertEquals(Float.floatToRawIntBits(3.0f), Float.floatToRawIntBits(reader.readFloat()));
+ }
+
+ /**
+ * Hand-craft double ALP bytes and verify reader independence.
+ * Values: {10.0, 20.0, 30.0}, vectorSize=8, e=0, f=0.
+ * encoded={10,20,30}, FOR min=10, deltas={0,10,20}, maxDelta=20, bitWidth=5.
+ */
+ @Test
+ public void testDoubleReaderWithHandCraftedBytes() throws Exception {
+ int numElements = 3;
+ int exponent = 0;
+ int factor = 0;
+ long frameOfReference = 10;
+ int bitWidth = 5; // ceil(log2(20+1)) = 5
+
+ org.apache.parquet.column.values.bitpacking.BytePackerForLong packer =
+ Packer.LITTLE_ENDIAN.newBytePackerForLong(5);
+ long[] deltas = {0, 10, 20, 0, 0, 0, 0, 0};
+ byte[] packed = new byte[5]; // bitWidth bytes for group of 8
+ packer.pack8Values(deltas, 0, packed, 0);
+ int packedSize = (3 * 5 + 7) / 8; // = 2 bytes
+
+ int vectorDataSize = 4 + 9 + packedSize; // AlpInfo + DoubleForInfo + packed
+ int offsetArraySize = 4;
+
+ ByteBuffer page =
+ ByteBuffer.allocate(7 + offsetArraySize + vectorDataSize).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) 3); // log2(8)
+ page.putInt(numElements);
+
+ // Offset array
+ page.putInt(offsetArraySize);
+
+ // AlpInfo
+ page.put((byte) exponent);
+ page.put((byte) factor);
+ page.putShort((short) 0); // no exceptions
+
+ // ForInfo (9 bytes for double)
+ page.putLong(frameOfReference);
+ page.put((byte) bitWidth);
+
+ // Packed data
+ page.put(packed, 0, packedSize);
+
+ page.flip();
+
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ assertEquals(Double.doubleToRawLongBits(10.0), Double.doubleToRawLongBits(reader.readDouble()));
+ assertEquals(Double.doubleToRawLongBits(20.0), Double.doubleToRawLongBits(reader.readDouble()));
+ assertEquals(Double.doubleToRawLongBits(30.0), Double.doubleToRawLongBits(reader.readDouble()));
+ }
+
+ // ========== Writer Output → Hand-Verified Bytes ==========
+
+ /**
+ * Write known values, then verify the exact binary output against hand computation.
+ * This verifies the writer independently of the reader.
+ *
+ *
Input: {1.0f, 2.0f, 3.0f} with vectorSize=8.
+ * Expected: e=0, f=0, encoded={1,2,3}, FOR min=1, deltas={0,1,2},
+ * maxDelta=2, bitWidth=2. Packed size = ceil(3*2/8) = 1 byte.
+ */
+ @Test
+ public void testWriterOutputExactBytes() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), 8);
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+ writer.writeFloat(3.0f);
+
+ byte[] bytes = writer.getBytes().toByteArray();
+ ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ assertEquals(0, buf.get() & 0xFF); // compression_mode
+ assertEquals(0, buf.get() & 0xFF); // integer_encoding
+ assertEquals(3, buf.get() & 0xFF); // log2(8) = 3
+ assertEquals(3, buf.getInt()); // num_elements
+
+ // Offset array (1 vector)
+ int offset0 = buf.getInt();
+ assertEquals(4, offset0); // 1 * 4 = past offset array
+
+ // AlpInfo
+ int exp = buf.get() & 0xFF;
+ int fac = buf.get() & 0xFF;
+ int numExc = buf.getShort() & 0xFFFF;
+ assertEquals(0, numExc);
+
+ // ForInfo
+ int forRef = buf.getInt();
+ int bw = buf.get() & 0xFF;
+
+ // Verify: for e=0, f=0: multiplier=1.0
+ // encoded = {1, 2, 3}, min = 1, deltas = {0, 1, 2}, bitWidth = 2
+ assertEquals(1, forRef); // frame of reference = 1
+ assertEquals(2, bw); // bitWidth for max delta 2
+
+ // Packed data: ceil(3 * 2 / 8) = 1 byte
+ // deltas {0, 1, 2} at bitWidth=2:
+ // Use BytePacker to compute expected packed byte
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(2);
+ int[] expectedDeltas = {0, 1, 2, 0, 0, 0, 0, 0};
+ byte[] expectedPacked = new byte[2]; // bitWidth=2
+ packer.pack8Values(expectedDeltas, 0, expectedPacked, 0);
+
+ byte actualPackedByte = buf.get();
+ assertEquals("packed byte", expectedPacked[0], actualPackedByte);
+
+ // Should have consumed all bytes
+ assertEquals("all bytes consumed", bytes.length, buf.position());
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== NaN Bit Pattern Preservation ==========
+
+ /**
+ * Verify that different NaN bit patterns survive the round-trip exactly.
+ * Java allows multiple NaN representations. Our exception handling must
+ * preserve the exact bit pattern, not normalize to Float.NaN.
+ */
+ @Test
+ public void testNaNBitPatternPreservation() throws Exception {
+ // Standard NaN
+ float standardNaN = Float.NaN; // 0x7FC00000
+ // Signaling NaN (different bit pattern)
+ float signalingNaN = Float.intBitsToFloat(0x7F800001);
+ // Negative NaN
+ float negativeNaN = Float.intBitsToFloat(0xFFC00000);
+ // Another NaN variant
+ float customNaN = Float.intBitsToFloat(0x7FFFFFFF);
+
+ assertTrue(Float.isNaN(standardNaN));
+ assertTrue(Float.isNaN(signalingNaN));
+ assertTrue(Float.isNaN(negativeNaN));
+ assertTrue(Float.isNaN(customNaN));
+
+ float[] values = {1.0f, standardNaN, 2.0f, signalingNaN, 3.0f, negativeNaN, 4.0f, customNaN};
+
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ float actual = reader.readFloat();
+ assertEquals(
+ "Bit pattern mismatch at index " + i + " (0x"
+ + Integer.toHexString(Float.floatToRawIntBits(values[i])) + " vs 0x"
+ + Integer.toHexString(Float.floatToRawIntBits(actual)) + ")",
+ Float.floatToRawIntBits(values[i]),
+ Float.floatToRawIntBits(actual));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ /**
+ * Same test for double NaN bit patterns.
+ */
+ @Test
+ public void testDoubleNaNBitPatternPreservation() throws Exception {
+ double standardNaN = Double.NaN;
+ double signalingNaN = Double.longBitsToDouble(0x7FF0000000000001L);
+ double negativeNaN = Double.longBitsToDouble(0xFFF8000000000000L);
+ double customNaN = Double.longBitsToDouble(0x7FFFFFFFFFFFFFFFL);
+
+ double[] values = {1.0, standardNaN, 2.0, signalingNaN, 3.0, negativeNaN, 4.0, customNaN};
+
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(512, 512, new DirectByteBufferAllocator());
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ double actual = reader.readDouble();
+ assertEquals(
+ "Bit pattern mismatch at index " + i + " (0x"
+ + Long.toHexString(Double.doubleToRawLongBits(values[i])) + " vs 0x"
+ + Long.toHexString(Double.doubleToRawLongBits(actual)) + ")",
+ Double.doubleToRawLongBits(values[i]),
+ Double.doubleToRawLongBits(actual));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Negative Zero Bit-Exact Roundtrip ==========
+
+ /**
+ * Verify that -0.0f roundtrips as -0.0f (bit pattern 0x80000000),
+ * not as +0.0f (bit pattern 0x00000000).
+ */
+ @Test
+ public void testNegativeZeroBitExact() throws Exception {
+ float negZero = -0.0f;
+ float posZero = 0.0f;
+
+ // Sanity: they are == but have different bit patterns
+ assertTrue(negZero == posZero);
+ assertNotEquals(Float.floatToRawIntBits(negZero), Float.floatToRawIntBits(posZero));
+
+ float[] values = {posZero, negZero, 1.0f, negZero};
+
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ float actual = reader.readFloat();
+ assertEquals(
+ "Bit pattern at index " + i,
+ Float.floatToRawIntBits(values[i]),
+ Float.floatToRawIntBits(actual));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Extreme Float Values ==========
+
+ /**
+ * Test subnormal floats, Float.MIN_VALUE, Float.MAX_VALUE, Float.MIN_NORMAL.
+ * These should all either encode losslessly or be stored as exceptions.
+ */
+ @Test
+ public void testExtremeFloatValues() throws Exception {
+ float[] values = {
+ Float.MIN_VALUE, // smallest positive subnormal: 1.4e-45
+ Float.MIN_NORMAL, // smallest positive normal: 1.17549435e-38
+ Float.MAX_VALUE, // 3.4028235e38
+ -Float.MAX_VALUE, // most negative
+ 1.17549435e-38f, // near MIN_NORMAL
+ 3.4028234e38f, // near MAX_VALUE
+ 1.0e-10f, // very small positive
+ 1.0e10f, // large positive
+ };
+ roundTripFloat(values);
+ }
+
+ /**
+ * Test extreme double values.
+ */
+ @Test
+ public void testExtremeDoubleValues() throws Exception {
+ double[] values = {
+ Double.MIN_VALUE, // smallest positive subnormal
+ Double.MIN_NORMAL, // smallest positive normal
+ Double.MAX_VALUE, // largest positive
+ -Double.MAX_VALUE, // most negative
+ 1.0e-100, // very small
+ 1.0e100, // very large
+ };
+ roundTripDouble(values);
+ }
+
+ // ========== Preset Caching Under Distribution Change ==========
+
+ /**
+ * Write >8 vectors where the optimal (e,f) changes after the sampling phase.
+ * First 8 vectors: monetary data (e=2, f=0 optimal).
+ * Remaining vectors: large integers (e=0, f=0 optimal).
+ * Verify all values survive round-trip despite the distribution change.
+ */
+ @Test
+ public void testPresetCachingWithDistributionChange() throws Exception {
+ int vectorSize = 8; // small vectors to get to 8 quickly
+ int samplingVectors = 8;
+ int postSamplingVectors = 4;
+ int totalValues = vectorSize * (samplingVectors + postSamplingVectors);
+
+ float[] values = new float[totalValues];
+
+ // First 8 vectors: decimal monetary values (best with e=2, f=0)
+ for (int i = 0; i < vectorSize * samplingVectors; i++) {
+ values[i] = 10.00f + (i % 100) * 0.01f; // 10.00, 10.01, 10.02, ...
+ }
+
+ // Last 4 vectors: whole numbers (best with e=0, f=0)
+ for (int i = vectorSize * samplingVectors; i < totalValues; i++) {
+ values[i] = (float) (i * 1000);
+ }
+
+ roundTripFloat(values, vectorSize);
+ }
+
+ /**
+ * Same test for doubles.
+ */
+ @Test
+ public void testDoublePresetCachingWithDistributionChange() throws Exception {
+ int vectorSize = 8;
+ int totalValues = vectorSize * 12; // 8 sampling + 4 post
+
+ double[] values = new double[totalValues];
+
+ for (int i = 0; i < vectorSize * 8; i++) {
+ values[i] = 10.00 + (i % 100) * 0.01;
+ }
+ for (int i = vectorSize * 8; i < totalValues; i++) {
+ values[i] = (double) (i * 1000);
+ }
+
+ roundTripDouble(values, vectorSize);
+ }
+
+ // ========== Multi-Vector Hand-Crafted Binary ==========
+
+ /**
+ * Hand-craft a 2-vector page to verify offset array navigation works.
+ * 10 values with vectorSize=8: vector 0 has values 1-8, vector 1 has 9-10.
+ */
+ @Test
+ public void testMultiVectorHandCrafted() throws Exception {
+ int vectorSize = 8;
+ int numElements = 10;
+ int numVectors = 2;
+
+ // Vector 0: {1,2,3,4,5,6,7,8} → e=0,f=0, encoded={1..8}, FOR min=1, deltas={0..7}
+ // maxDelta=7, bitWidth=3. Packed: 8 values at 3 bits = 3 bytes exactly.
+ int v0BitWidth = 3;
+ BytePacker packer3 = Packer.LITTLE_ENDIAN.newBytePacker(3);
+ int[] v0Deltas = {0, 1, 2, 3, 4, 5, 6, 7};
+ byte[] v0Packed = new byte[3];
+ packer3.pack8Values(v0Deltas, 0, v0Packed, 0);
+ int v0PackedSize = 3; // (8*3+7)/8 = 3
+
+ // Vector 1: {9, 10} → e=0,f=0, encoded={9,10}, FOR min=9, deltas={0,1}
+ // maxDelta=1, bitWidth=1. Packed: ceil(2*1/8) = 1 byte.
+ int v1BitWidth = 1;
+ BytePacker packer1 = Packer.LITTLE_ENDIAN.newBytePacker(1);
+ int[] v1Deltas = {0, 1, 0, 0, 0, 0, 0, 0};
+ byte[] v1Packed = new byte[1];
+ packer1.pack8Values(v1Deltas, 0, v1Packed, 0);
+ int v1PackedSize = 1; // (2*1+7)/8 = 1
+
+ int v0DataSize = 4 + 5 + v0PackedSize; // 12
+ int v1DataSize = 4 + 5 + v1PackedSize; // 10
+ int offsetArraySize = numVectors * 4; // 8
+
+ ByteBuffer page = ByteBuffer.allocate(7 + offsetArraySize + v0DataSize + v1DataSize)
+ .order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) 3); // log2(8)
+ page.putInt(numElements);
+
+ // Offset array
+ page.putInt(offsetArraySize); // vector 0 starts at 8
+ page.putInt(offsetArraySize + v0DataSize); // vector 1 starts at 8+12=20
+
+ // Vector 0
+ page.put((byte) 0); // exponent
+ page.put((byte) 0); // factor
+ page.putShort((short) 0); // exceptions
+ page.putInt(1); // FOR = 1
+ page.put((byte) v0BitWidth);
+ page.put(v0Packed, 0, v0PackedSize);
+
+ // Vector 1
+ page.put((byte) 0); // exponent
+ page.put((byte) 0); // factor
+ page.putShort((short) 0); // exceptions
+ page.putInt(9); // FOR = 9
+ page.put((byte) v1BitWidth);
+ page.put(v1Packed, 0, v1PackedSize);
+
+ page.flip();
+
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ for (int i = 0; i < numElements; i++) {
+ float expected = (float) (i + 1);
+ float actual = reader.readFloat();
+ assertEquals(
+ "Value mismatch at index " + i, Float.floatToRawIntBits(expected), Float.floatToRawIntBits(actual));
+ }
+ }
+
+ // ========== Skip Over Entire Vector Without Decoding ==========
+
+ /**
+ * Verify that skipping an entire vector and reading the next one works.
+ * Uses hand-crafted bytes to ensure independence from writer.
+ */
+ @Test
+ public void testSkipEntireVectorHandCrafted() throws Exception {
+ // Reuse the multi-vector setup: {1..8} in vector 0, {9, 10} in vector 1
+ int vectorSize = 8;
+ int numElements = 10;
+ int numVectors = 2;
+
+ BytePacker packer3 = Packer.LITTLE_ENDIAN.newBytePacker(3);
+ int[] v0Deltas = {0, 1, 2, 3, 4, 5, 6, 7};
+ byte[] v0Packed = new byte[3];
+ packer3.pack8Values(v0Deltas, 0, v0Packed, 0);
+
+ BytePacker packer1 = Packer.LITTLE_ENDIAN.newBytePacker(1);
+ int[] v1Deltas = {0, 1, 0, 0, 0, 0, 0, 0};
+ byte[] v1Packed = new byte[1];
+ packer1.pack8Values(v1Deltas, 0, v1Packed, 0);
+
+ int v0DataSize = 4 + 5 + 3;
+ int v1DataSize = 4 + 5 + 1;
+ int offsetArraySize = numVectors * 4;
+
+ ByteBuffer page = ByteBuffer.allocate(7 + offsetArraySize + v0DataSize + v1DataSize)
+ .order(ByteOrder.LITTLE_ENDIAN);
+
+ page.put((byte) 0).put((byte) 0).put((byte) 3);
+ page.putInt(numElements);
+ page.putInt(offsetArraySize);
+ page.putInt(offsetArraySize + v0DataSize);
+
+ // Vector 0
+ page.put((byte) 0).put((byte) 0).putShort((short) 0);
+ page.putInt(1).put((byte) 3);
+ page.put(v0Packed, 0, 3);
+
+ // Vector 1
+ page.put((byte) 0).put((byte) 0).putShort((short) 0);
+ page.putInt(9).put((byte) 1);
+ page.put(v1Packed, 0, 1);
+
+ page.flip();
+
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ // Skip entire vector 0 (8 values)
+ reader.skip(8);
+
+ // Read from vector 1
+ assertEquals(Float.floatToRawIntBits(9.0f), Float.floatToRawIntBits(reader.readFloat()));
+ assertEquals(Float.floatToRawIntBits(10.0f), Float.floatToRawIntBits(reader.readFloat()));
+ }
+
+ // ========== Vector Size Validation ==========
+
+ /**
+ * Verify that vectorSize=65536 is rejected because num_exceptions (uint16)
+ * cannot represent 65536, which would cause silent data corruption if all
+ * values in a vector are exceptions.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testVectorSize65536Rejected() throws Exception {
+ new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), 65536);
+ }
+
+ /**
+ * Verify that vectorSize=32768 (max allowed) works correctly.
+ */
+ @Test
+ public void testVectorSize32768Works() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), 32768);
+ // Write a small number of values (partial vector)
+ for (int i = 0; i < 10; i++) {
+ writer.writeFloat(i * 1.0f);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(10, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < 10; i++) {
+ assertEquals(Float.floatToRawIntBits(i * 1.0f), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Factor > 0 Tests ==========
+
+ /**
+ * Data where the best (e,f) combo has factor > 0.
+ * Values like 1234.56 have digits on both sides of the decimal point.
+ * With e=4,f=2: 1234.56 * 10000 / 100 = 123456 — exact integer.
+ * This exercises the division path in the encode/decode formula.
+ */
+ @Test
+ public void testDoubleFactorGreaterThanZero() throws Exception {
+ // These values are designed so the best combo has f > 0
+ double[] values = new double[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 1000.0 + i * 0.01; // 1000.00, 1000.01, ...
+ }
+
+ // Verify the best params actually have f > 0 or f == 0 with low exceptions
+ AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestDoubleParams(values, 0, values.length);
+ // Regardless of the chosen f, the roundtrip must be lossless
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatFactorGreaterThanZero() throws Exception {
+ float[] values = new float[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 1000.0f + i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== Overflow Boundary Tests ==========
+
+ /**
+ * Values near the encoding limits that test the overflow check.
+ * The old code used Long.MAX_VALUE (which rounds to 2^63 as double),
+ * the fix uses ENCODING_UPPER_LIMIT = 9223372036854774784.0.
+ */
+ @Test
+ public void testDoubleNearOverflowBoundary() throws Exception {
+ double[] values = {
+ 9.2e18, // near Long.MAX_VALUE
+ -9.2e18, // near Long.MIN_VALUE
+ 1.7e308, // near Double.MAX_VALUE
+ -1.7e308, // near -Double.MAX_VALUE
+ 4.9e-324, // Double.MIN_VALUE (subnormal)
+ 1.0e18, // large but in range
+ -1.0e18, // large negative but in range
+ 0.0, // normal
+ 1.23, // normal
+ };
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatNearOverflowBoundary() throws Exception {
+ float[] values = {
+ 2.1e9f, // near Integer.MAX_VALUE
+ -2.1e9f, // near Integer.MIN_VALUE
+ 3.4e38f, // near Float.MAX_VALUE
+ -3.4e38f, // near -Float.MAX_VALUE
+ 1.4e-45f, // Float.MIN_VALUE (subnormal)
+ 1.0e9f, // large but in range
+ 0.0f, // normal
+ 1.23f, // normal
+ };
+ roundTripFloat(values);
+ }
+
+ // ========== Large Scale Stress Test ==========
+
+ @Test
+ public void testDoubleLargeScale100K() throws Exception {
+ Random rand = new Random(12345);
+ double[] values = new double[100_000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextDouble() * 10000.0 - 5000.0;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatLargeScale100K() throws Exception {
+ Random rand = new Random(12345);
+ float[] values = new float[100_000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextFloat() * 10000.0f - 5000.0f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== All Exceptions in Large Vector ==========
+
+ /**
+ * A full vector where every value is an exception (NaN/Inf/-0.0).
+ * This exercises the case where placeholder search finds no valid value.
+ */
+ @Test
+ public void testFloatEntireVectorAllExceptions() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE + 5]; // partial second vector
+ for (int i = 0; i < values.length; i++) {
+ switch (i % 4) {
+ case 0:
+ values[i] = Float.NaN;
+ break;
+ case 1:
+ values[i] = Float.POSITIVE_INFINITY;
+ break;
+ case 2:
+ values[i] = Float.NEGATIVE_INFINITY;
+ break;
+ case 3:
+ values[i] = -0.0f;
+ break;
+ }
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testDoubleEntireVectorAllExceptions() throws Exception {
+ double[] values = new double[DEFAULT_VECTOR_SIZE + 5];
+ for (int i = 0; i < values.length; i++) {
+ switch (i % 4) {
+ case 0:
+ values[i] = Double.NaN;
+ break;
+ case 1:
+ values[i] = Double.POSITIVE_INFINITY;
+ break;
+ case 2:
+ values[i] = Double.NEGATIVE_INFINITY;
+ break;
+ case 3:
+ values[i] = -0.0;
+ break;
+ }
+ }
+ roundTripDouble(values);
+ }
+
+ // ========== Verify Encoder Produces Expected Values ==========
+
+ /**
+ * Manually verify that the encoder produces the exact integer values
+ * we expect for known inputs. This is independent of the writer/reader.
+ */
+ @Test
+ public void testEncoderProducesExpectedValues() {
+ // 1.23f * 100 = 123
+ assertEquals(123, AlpEncoderDecoder.encodeFloat(1.23f, 2, 0));
+ // 19.99f * 100 = 1999
+ assertEquals(1999, AlpEncoderDecoder.encodeFloat(19.99f, 2, 0));
+ // -5.0f * 10 = -50
+ assertEquals(-50, AlpEncoderDecoder.encodeFloat(-5.0f, 1, 0));
+ // 0.0f * anything = 0
+ assertEquals(0, AlpEncoderDecoder.encodeFloat(0.0f, 5, 0));
+ // 42.0f * 1 = 42
+ assertEquals(42, AlpEncoderDecoder.encodeFloat(42.0f, 0, 0));
+ // 1.5f * 10 = 15
+ assertEquals(15, AlpEncoderDecoder.encodeFloat(1.5f, 1, 0));
+
+ // Double path
+ assertEquals(123L, AlpEncoderDecoder.encodeDouble(1.23, 2, 0));
+ assertEquals(1999L, AlpEncoderDecoder.encodeDouble(19.99, 2, 0));
+ assertEquals(-50L, AlpEncoderDecoder.encodeDouble(-5.0, 1, 0));
+ assertEquals(0L, AlpEncoderDecoder.encodeDouble(0.0, 5, 0));
+ assertEquals(42L, AlpEncoderDecoder.encodeDouble(42.0, 0, 0));
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java
index 17f786c4a5..ffffb07110 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java
@@ -33,6 +33,7 @@
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.alp.AlpValuesWriter;
import org.apache.parquet.column.values.bytestreamsplit.ByteStreamSplitValuesWriter;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong;
@@ -461,6 +462,42 @@ public void testDouble_V2_WithByteStreamSplitAndDictionary() {
testFloatingPoint_WithByteStreamSplitAndDictionary(PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_2_0);
}
+ @Test
+ public void testFloat_V1_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.FLOAT, WriterVersion.PARQUET_1_0, AlpValuesWriter.FloatAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testDouble_V1_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_1_0, AlpValuesWriter.DoubleAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testFloat_V2_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.FLOAT, WriterVersion.PARQUET_2_0, AlpValuesWriter.FloatAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testDouble_V2_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_2_0, AlpValuesWriter.DoubleAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testFloat_Alp_TakesPrecedenceOverByteStreamSplit() {
+ testAlpTakesPrecedenceOverByteStreamSplit(
+ PrimitiveTypeName.FLOAT, WriterVersion.PARQUET_1_0, AlpValuesWriter.FloatAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testDouble_Alp_TakesPrecedenceOverByteStreamSplit() {
+ testAlpTakesPrecedenceOverByteStreamSplit(
+ PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_2_0, AlpValuesWriter.DoubleAlpValuesWriter.class);
+ }
+
@Test
public void testColumnWiseDictionaryWithFalseDefault() {
ValuesWriterFactory factory = getDefaultFactory(
@@ -632,6 +669,50 @@ private void testFloatingPoint_WithByteStreamSplitAndDictionary(
PlainValuesWriter.class);
}
+ private void testFloatingPoint_WithAlp(
+ PrimitiveTypeName typeName, WriterVersion writerVersion, Class alpWriterClass) {
+ // Cross-column ALP without a dictionary: the ALP writer is produced directly.
+ doTestValueWriter(
+ createColumnDescriptor(typeName),
+ ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withDictionaryEncoding(false)
+ .withAlpEncoding(true)
+ .build(),
+ alpWriterClass);
+ // Cross-column ALP with a dictionary: the ALP writer is the dictionary's fallback.
+ doTestValueWriter(
+ createColumnDescriptor(typeName),
+ ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withAlpEncoding(true)
+ .build(),
+ DictionaryValuesWriter.class,
+ alpWriterClass);
+ // Per-column ALP: only colA opts in, colB keeps the plain fallback.
+ ParquetProperties properties = ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withDictionaryEncoding(false)
+ .withAlpEncoding("colA", true)
+ .build();
+ doTestValueWriter(createColumnDescriptor(typeName, "colA"), properties, alpWriterClass);
+ doTestValueWriter(createColumnDescriptor(typeName, "colB"), properties, PlainValuesWriter.class);
+ }
+
+ private void testAlpTakesPrecedenceOverByteStreamSplit(
+ PrimitiveTypeName typeName, WriterVersion writerVersion, Class alpWriterClass) {
+ // When both ALP and byte-stream-split are enabled the factory prefers ALP.
+ doTestValueWriter(
+ createColumnDescriptor(typeName),
+ ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withDictionaryEncoding(false)
+ .withAlpEncoding(true)
+ .withByteStreamSplitEncoding(true)
+ .build(),
+ alpWriterClass);
+ }
+
private void validateFactory(
ValuesWriterFactory factory,
PrimitiveTypeName typeName,
diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml
index 3b31afca90..ef3119da93 100644
--- a/parquet-format-structures/pom.xml
+++ b/parquet-format-structures/pom.xml
@@ -66,6 +66,35 @@
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+ ${exec-maven-plugin.version}
+
+
+ patch-alp-encoding
+ process-sources
+
+ exec
+
+
+ perl
+
+ ${basedir}/src/main/perl/patch-alp-encoding.pl
+ ${project.build.directory}/generated-sources/thrift/org/apache/parquet/format/Encoding.java
+
+
+
+
+
org.apache.thrift
diff --git a/parquet-format-structures/src/main/perl/patch-alp-encoding.pl b/parquet-format-structures/src/main/perl/patch-alp-encoding.pl
new file mode 100644
index 0000000000..7f80ecfd6e
--- /dev/null
+++ b/parquet-format-structures/src/main/perl/patch-alp-encoding.pl
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Patch the generated Encoding.java to add ALP(10) support.
+# ALP (Adaptive Lossless floating-Point) uses encoding value 10 (next after BYTE_STREAM_SPLIT=9).
+# This patch should be removed once parquet-format includes ALP and parquet-java updates its dep.
+use strict; use warnings;
+my $file = $ARGV[0] or die "Usage: $0 \n";
+open(my $fh, '<', $file) or die "Cannot read $file: $!";
+my $content = do { local $/; <$fh> };
+close($fh);
+exit 0 if $content =~ /ALP/; # already patched
+
+# 1. Add ALP to the enum values list
+$content =~ s/BYTE_STREAM_SPLIT\(9\);/BYTE_STREAM_SPLIT(9),\n\n \/** ALP (Adaptive Lossless floating-Point) encoding for FLOAT and DOUBLE types.\n * Encoding value 10. See https:\/\/github.com\/cwida\/ALP\n *\/\n ALP(10);/;
+
+# 2. Add ALP to the findByValue switch
+$content =~ s/(case 9:\s*\n\s*return BYTE_STREAM_SPLIT;)/$1\n case 10:\n return ALP;/;
+
+open(my $out, '>', $file) or die "Cannot write $file: $!";
+print $out $content;
+close($out);
+print "Patched $file with ALP(10)\n";
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
index 7dcbb3188c..0f84784076 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
@@ -737,6 +737,26 @@ public SELF withByteStreamSplitEncoding(String columnPath, boolean enableByteStr
return self();
}
+ public SELF withAlpEncoding(boolean enableAlp) {
+ encodingPropsBuilder.withAlpEncoding(enableAlp);
+ return self();
+ }
+
+ public SELF withAlpEncoding(String columnPath, boolean enableAlp) {
+ encodingPropsBuilder.withAlpEncoding(columnPath, enableAlp);
+ return self();
+ }
+
+ public SELF withAlpVectorSize(int vectorSize) {
+ encodingPropsBuilder.withAlpVectorSize(vectorSize);
+ return self();
+ }
+
+ public SELF withAlpVectorSize(String columnPath, int vectorSize) {
+ encodingPropsBuilder.withAlpVectorSize(columnPath, vectorSize);
+ return self();
+ }
+
/**
* Enable or disable dictionary encoding of the specified column for the constructed writer.
*
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index 8d778f7b91..4e9038b7af 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -471,6 +471,10 @@ public void testLogicalToConvertedTypeConversion() {
public void testEnumEquivalence() {
ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
for (org.apache.parquet.column.Encoding encoding : org.apache.parquet.column.Encoding.values()) {
+ // Skip ALP encoding as it's not yet in the parquet-format specification
+ if (encoding == org.apache.parquet.column.Encoding.ALP) {
+ continue;
+ }
assertEquals(
encoding, parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding)));
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadAlp.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadAlp.java
new file mode 100644
index 0000000000..67742984a1
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadAlp.java
@@ -0,0 +1,1399 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.hadoop;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.ParquetProperties.WriterVersion;
+import org.apache.parquet.column.page.PageReadStore;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.example.data.simple.convert.GroupRecordConverter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.io.ColumnIOFactory;
+import org.apache.parquet.io.LocalInputFile;
+import org.apache.parquet.io.LocalOutputFile;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.io.RecordReader;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.MessageTypeParser;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Cross-compatibility test for ALP (Adaptive Lossless floating-Point) encoding.
+ *
+ * Reads ALP-encoded parquet files generated by Arrow C++ and verifies that the Java
+ * implementation decodes them correctly.
+ *
+ *
Set ALP_TEST_FILE (env var or system property) to a single file, or use the
+ * ALP_TEST_DATA_DIR property pointing to the alp-test-data/ directory.
+ *
+ * @see Arrow C++ ALP PR
+ * @see parquet-testing ALP PR
+ */
+public class TestInterOpReadAlp {
+ private static final Logger LOG = LoggerFactory.getLogger(TestInterOpReadAlp.class);
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ private static final String[] CPP_DOUBLE_FILES = {"alp_spotify1.parquet", "alp_arade.parquet"};
+ private static final String[] CPP_FLOAT_FILES = {"alp_float_spotify1.parquet", "alp_float_arade.parquet"};
+
+ private java.nio.file.Path getTestDataDir() {
+ String dir = System.getProperty("ALP_TEST_DATA_DIR");
+ if (dir == null) dir = System.getenv("ALP_TEST_DATA_DIR");
+ if (dir != null && new File(dir).isDirectory()) return Paths.get(dir);
+ // Default: alp-test-data/ relative to project root (two levels up from target/test-classes)
+ java.nio.file.Path candidate = Paths.get(System.getProperty("user.dir")).resolve("alp-test-data");
+ return candidate.toFile().isDirectory() ? candidate : null;
+ }
+
+ private java.nio.file.Path getSingleTestFile() {
+ String filePath = System.getProperty("ALP_TEST_FILE");
+ if (filePath == null) filePath = System.getenv("ALP_TEST_FILE");
+ if (filePath != null && new File(filePath).exists()) return Paths.get(filePath);
+ return null;
+ }
+
+ /** Read all rows from a parquet file using LocalInputFile (no Hadoop FileSystem). */
+ private List readAllRows(java.nio.file.Path filePath) throws IOException {
+ List rows = new ArrayList<>();
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(filePath))) {
+ ParquetMetadata footer = reader.getFooter();
+ MessageType schema = footer.getFileMetaData().getSchema();
+ MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
+ PageReadStore pages;
+ while ((pages = reader.readNextRowGroup()) != null) {
+ long rowCount = pages.getRowCount();
+ RecordReader recordReader = columnIO.getRecordReader(pages, new GroupRecordConverter(schema));
+ for (long i = 0; i < rowCount; i++) {
+ rows.add(recordReader.read());
+ }
+ }
+ }
+ return rows;
+ }
+
+ @Test
+ public void testReadSingleAlpFile() throws IOException {
+ java.nio.file.Path file = getSingleTestFile();
+ assumeTrue("ALP_TEST_FILE not set or file does not exist, skipping", file != null);
+ List rows = readAllRows(file);
+ assertTrue("Expected at least one row in " + file, rows.size() > 0);
+ LOG.info("testReadSingleAlpFile: read {} rows from {}", rows.size(), file.getFileName());
+ }
+
+ @Test
+ public void testReadCppDoubleFiles() throws IOException {
+ java.nio.file.Path dir = getTestDataDir();
+ assumeTrue("alp-test-data/ directory not found, skipping", dir != null);
+ for (String filename : CPP_DOUBLE_FILES) {
+ java.nio.file.Path file = dir.resolve(filename);
+ assumeTrue("File not found: " + file, file.toFile().exists());
+ List rows = readAllRows(file);
+ assertTrue("Expected rows in " + filename, rows.size() > 0);
+ LOG.info("testReadCppDoubleFiles: {} → {} rows OK", filename, rows.size());
+ }
+ }
+
+ @Test
+ public void testReadCppFloatFiles() throws IOException {
+ java.nio.file.Path dir = getTestDataDir();
+ assumeTrue("alp-test-data/ directory not found, skipping", dir != null);
+ for (String filename : CPP_FLOAT_FILES) {
+ java.nio.file.Path file = dir.resolve(filename);
+ assumeTrue("File not found: " + file, file.toFile().exists());
+ List rows = readAllRows(file);
+ assertTrue("Expected rows in " + filename, rows.size() > 0);
+ LOG.info("testReadCppFloatFiles: {} → {} rows OK", filename, rows.size());
+ }
+ }
+
+ /** Verify no NaN/Inf corruption in float/double columns across all test files. */
+ @Test
+ public void testNoCorruptionInCppFiles() throws IOException {
+ java.nio.file.Path dir = getTestDataDir();
+ assumeTrue("alp-test-data/ directory not found, skipping", dir != null);
+ String[] allFiles = {
+ "alp_spotify1.parquet", "alp_arade.parquet", "alp_float_spotify1.parquet", "alp_float_arade.parquet"
+ };
+ for (String filename : allFiles) {
+ java.nio.file.Path file = dir.resolve(filename);
+ if (!file.toFile().exists()) continue;
+ int nanCount = 0;
+ int infCount = 0;
+ int totalValues = 0;
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(file))) {
+ MessageType schema = reader.getFooter().getFileMetaData().getSchema();
+ MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
+ PageReadStore pages;
+ while ((pages = reader.readNextRowGroup()) != null) {
+ RecordReader recordReader =
+ columnIO.getRecordReader(pages, new GroupRecordConverter(schema));
+ for (long i = 0; i < pages.getRowCount(); i++) {
+ Group row = recordReader.read();
+ for (int f = 0; f < schema.getFieldCount(); f++) {
+ PrimitiveType.PrimitiveTypeName type =
+ schema.getType(f).asPrimitiveType().getPrimitiveTypeName();
+ try {
+ if (type == PrimitiveType.PrimitiveTypeName.FLOAT) {
+ float v = row.getFloat(f, 0);
+ totalValues++;
+ if (Float.isNaN(v)) nanCount++;
+ if (Float.isInfinite(v)) infCount++;
+ } else if (type == PrimitiveType.PrimitiveTypeName.DOUBLE) {
+ double v = row.getDouble(f, 0);
+ totalValues++;
+ if (Double.isNaN(v)) nanCount++;
+ if (Double.isInfinite(v)) infCount++;
+ }
+ } catch (Exception ignored) {
+ }
+ }
+ }
+ }
+ }
+ LOG.info("{}: {} values, {} NaN, {} Inf", filename, totalValues, nanCount, infCount);
+ assertEquals("Unexpected NaN in " + filename, 0, nanCount);
+ assertEquals("Unexpected Inf in " + filename, 0, infCount);
+ }
+ }
+
+ /** Schema inspection: log encoding types used in each column. */
+ @Test
+ public void testLogSchemaAndEncodings() throws IOException {
+ java.nio.file.Path dir = getTestDataDir();
+ assumeTrue("alp-test-data/ directory not found, skipping", dir != null);
+ String[] allFiles = {
+ "alp_spotify1.parquet", "alp_arade.parquet", "alp_float_spotify1.parquet", "alp_float_arade.parquet"
+ };
+ for (String filename : allFiles) {
+ java.nio.file.Path file = dir.resolve(filename);
+ if (!file.toFile().exists()) continue;
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(file))) {
+ ParquetMetadata footer = reader.getFooter();
+ MessageType schema = footer.getFileMetaData().getSchema();
+ LOG.info("=== {} ===", filename);
+ LOG.info("Schema: {}", schema);
+ if (!footer.getBlocks().isEmpty()) {
+ for (org.apache.parquet.hadoop.metadata.ColumnChunkMetaData col :
+ footer.getBlocks().get(0).getColumns()) {
+ LOG.info(
+ " column={} encodings={} compression={}",
+ col.getPath(),
+ col.getEncodings(),
+ col.getCodec());
+ }
+ }
+ } catch (Exception e) {
+ LOG.warn("Could not read metadata for {}: {}", filename, e.getMessage());
+ }
+ }
+ // This test always passes — it's for inspection
+ assertTrue(true);
+ }
+
+ private static final String ALP_SCHEMA =
+ "message alp_interop { " + "required double double_col; " + "required float float_col; " + "}";
+
+ private static final double[] DOUBLE_VALUES = {
+ 1.23, 4.56, 7.89, 0.001, 1000.0, -3.14, 2.718281828, 9.99999, 0.123456789, 100.5
+ };
+ private static final float[] FLOAT_VALUES = {
+ 1.23f, 4.56f, 7.89f, 0.001f, 1000.0f, -3.14f, 2.718f, 9.999f, 0.1234f, 100.5f
+ };
+
+ /**
+ * Write an ALP-encoded file from Java using the given page version, then read it back and verify
+ * all double and float values round-trip exactly.
+ */
+ private void writeAndVerifyAlpFile(WriterVersion version) throws IOException {
+ MessageType schema = MessageTypeParser.parseMessageType(ALP_SCHEMA);
+ java.nio.file.Path outPath =
+ temp.newFolder().toPath().resolve("alp_java_" + version.name().toLowerCase() + ".parquet");
+
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(version)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < DOUBLE_VALUES.length; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("double_col", DOUBLE_VALUES[i]);
+ row.add("float_col", FLOAT_VALUES[i]);
+ writer.write(row);
+ }
+ }
+
+ List rows = readAllRows(outPath);
+ assertEquals("Row count mismatch for " + version, DOUBLE_VALUES.length, rows.size());
+ for (int i = 0; i < DOUBLE_VALUES.length; i++) {
+ assertEquals(
+ "double_col mismatch at row " + i + " for " + version,
+ DOUBLE_VALUES[i],
+ rows.get(i).getDouble("double_col", 0),
+ 0.0);
+ assertEquals(
+ "float_col mismatch at row " + i + " for " + version,
+ FLOAT_VALUES[i],
+ rows.get(i).getFloat("float_col", 0),
+ 0.0f);
+ }
+ LOG.info(
+ "writeAndVerifyAlpFile [{}]: wrote and read back {} rows from {}",
+ version,
+ rows.size(),
+ outPath.getFileName());
+ }
+
+ /**
+ * Java writes ALP-encoded floats/doubles using V1 (PARQUET_1_0) data pages and reads them back.
+ * Verifies the Java write path produces a valid file readable by this implementation.
+ */
+ @Test
+ public void testJavaWriteAlpV1Pages() throws IOException {
+ writeAndVerifyAlpFile(WriterVersion.PARQUET_1_0);
+ }
+
+ /**
+ * Writes >4096 rows with vectorSize=4096 so multiple full vectors are flushed, then verifies
+ * the file round-trips exactly. The reader pulls log_vector_size from the on-disk header to
+ * size its unpacking window, so a wrong header byte would surface as decode garbage —
+ * round-trip equality is sufficient proof that the configured vector size took effect.
+ */
+ @Test
+ public void testJavaWriteAlpCustomVectorSize() throws IOException {
+ MessageType schema = MessageTypeParser.parseMessageType(ALP_SCHEMA);
+ int rowCount = 4500; // crosses one full vector + partial tail at vectorSize=4096
+ double[] doubles = new double[rowCount];
+ float[] floats = new float[rowCount];
+ // 2-decimal sensor-like data — the ALP sweet spot, so few/no exceptions
+ for (int i = 0; i < rowCount; i++) {
+ doubles[i] = (i * 13L % 100000) / 100.0;
+ floats[i] = (float) ((i * 7L % 10000) / 100.0);
+ }
+
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_java_vs4096.parquet");
+
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withAlpVectorSize(4096)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < rowCount; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("double_col", doubles[i]);
+ row.add("float_col", floats[i]);
+ writer.write(row);
+ }
+ }
+
+ List rows = readAllRows(outPath);
+ assertEquals("Row count mismatch at vectorSize=4096", rowCount, rows.size());
+ for (int i = 0; i < rowCount; i++) {
+ assertEquals(
+ "double_col mismatch at row " + i, doubles[i], rows.get(i).getDouble("double_col", 0), 0.0);
+ assertEquals(
+ "float_col mismatch at row " + i, floats[i], rows.get(i).getFloat("float_col", 0), 0.0f);
+ }
+ LOG.info("testJavaWriteAlpCustomVectorSize: {} rows round-tripped at vectorSize=4096", rowCount);
+ }
+
+ /**
+ * Java writes ALP-encoded floats/doubles using V2 (PARQUET_2_0) data pages and reads them back.
+ * V2 page headers include the encoding value directly; ALP = 10 is supported via the build-time
+ * patch to the generated Encoding enum in parquet-format-structures.
+ */
+ @Test
+ public void testJavaWriteAlpV2Pages() throws IOException {
+ writeAndVerifyAlpFile(WriterVersion.PARQUET_2_0);
+ }
+
+ /**
+ * Writes ALP-encoded V1 and V2 parquet files from Java and verifies that pyarrow (Python Arrow)
+ * can read them back correctly. Skipped if python3/pyarrow is not available in the environment.
+ *
+ * This test addresses cross-language compatibility: Java writes, another implementation reads.
+ */
+ @Test
+ public void testJavaWrittenAlpFilesReadableByPyarrow() throws IOException, InterruptedException {
+ // Check python3 is available
+ Process check = new ProcessBuilder("python3", "-c", "import pyarrow.parquet")
+ .redirectErrorStream(true)
+ .start();
+ assumeTrue("python3/pyarrow not available, skipping cross-language test", check.waitFor() == 0);
+
+ for (WriterVersion version : new WriterVersion[] {WriterVersion.PARQUET_1_0, WriterVersion.PARQUET_2_0}) {
+ MessageType schema = MessageTypeParser.parseMessageType(ALP_SCHEMA);
+ java.nio.file.Path outPath = temp.newFolder()
+ .toPath()
+ .resolve("alp_java_xcompat_" + version.name().toLowerCase() + ".parquet");
+
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(version)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < DOUBLE_VALUES.length; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("double_col", DOUBLE_VALUES[i]);
+ row.add("float_col", FLOAT_VALUES[i]);
+ writer.write(row);
+ }
+ }
+
+ // Build a python snippet that reads the file and asserts row count and values
+ StringBuilder pyScript = new StringBuilder();
+ pyScript.append("import pyarrow.parquet as pq, sys\n");
+ pyScript.append("t = pq.read_table('")
+ .append(outPath.toAbsolutePath())
+ .append("')\n");
+ pyScript.append("assert len(t) == ")
+ .append(DOUBLE_VALUES.length)
+ .append(", f'row count mismatch: {len(t)}'\n");
+ pyScript.append("dc = t.column('double_col').to_pylist()\n");
+ pyScript.append("fc = t.column('float_col').to_pylist()\n");
+ for (int i = 0; i < DOUBLE_VALUES.length; i++) {
+ pyScript.append("assert abs(dc[")
+ .append(i)
+ .append("] - ")
+ .append(DOUBLE_VALUES[i])
+ .append(") < 1e-9, f'double mismatch at ")
+ .append(i)
+ .append(": {dc[")
+ .append(i)
+ .append("]}'\n");
+ pyScript.append("assert abs(fc[")
+ .append(i)
+ .append("] - ")
+ .append(FLOAT_VALUES[i])
+ .append(") < 1e-4, f'float mismatch at ")
+ .append(i)
+ .append(": {fc[")
+ .append(i)
+ .append("]}'\n");
+ }
+ pyScript.append(
+ "print('OK: " + version.name() + " " + DOUBLE_VALUES.length + " rows verified by pyarrow')\n");
+
+ Process proc = new ProcessBuilder("python3", "-c", pyScript.toString())
+ .redirectErrorStream(true)
+ .start();
+ String output = new String(proc.getInputStream().readAllBytes());
+ int exitCode = proc.waitFor();
+ LOG.info("pyarrow cross-compat [{}]: {}", version, output.trim());
+ // pyarrow may not yet support reading ALP (Arrow C++ PR #48345 in progress).
+ // Skip rather than fail so the test becomes a passing signal once pyarrow adds ALP support.
+ assumeTrue(
+ "pyarrow does not yet support reading ALP encoding (Arrow C++ PR #48345 pending): " + output,
+ !output.contains("Unknown encoding type"));
+ assertEquals("pyarrow failed to read Java-written ALP file (" + version + "): " + output, 0, exitCode);
+ }
+ }
+
+ // TODO: Uncomment once parquet-testing PR #100 merges
+ /*
+ @Test
+ public void testReadAlpFromParquetTesting() throws IOException {
+ // InterOpTester will auto-download from parquet-testing
+ }
+ */
+
+ // ---------------------------------------------------------------------------
+ // Fixture generator: reads the C++-written ALP files from parquet-testing
+ // PR #100 and re-encodes each as Java ALP at two vector sizes (1024 and 4096).
+ // Outputs land in ALP_OUTPUT_DIR (default: ${user.dir}/alp-java-generated/)
+ // and are intended for submission back to parquet-testing PR #100 so the
+ // C++/Rust/Go readers can verify cross-language compatibility against Java
+ // writes at both vector sizes.
+ // ---------------------------------------------------------------------------
+
+ private static final String[] SOURCE_FILES = {
+ "alp_spotify1.parquet", "alp_arade.parquet", "alp_float_spotify1.parquet", "alp_float_arade.parquet"
+ };
+ private static final int[] GENERATOR_VECTOR_SIZES = {1024, 4096};
+ private static final WriterVersion[] GENERATOR_PAGE_VERSIONS = {WriterVersion.PARQUET_1_0, WriterVersion.PARQUET_2_0
+ };
+
+ /**
+ * Resolves the explicit output directory from {@code ALP_OUTPUT_DIR} (system property or env var)
+ * and creates it if missing. Returns {@code null} when the variable is unset — callers should
+ * {@code assumeTrue} on the result so the test skips cleanly rather than writing stray files
+ * into the module tree (which would fail the RAT license check on subsequent {@code mvn test}
+ * runs in the same module).
+ */
+ private java.nio.file.Path getOutputDir() throws IOException {
+ String dir = System.getProperty("ALP_OUTPUT_DIR");
+ if (dir == null) dir = System.getenv("ALP_OUTPUT_DIR");
+ if (dir == null) return null;
+ java.nio.file.Path target = Paths.get(dir);
+ java.nio.file.Files.createDirectories(target);
+ return target;
+ }
+
+ /**
+ * Read all rows of the given source file and return them as a list of value-only Group rows
+ * keyed by the source's schema. Used for both the copy and the verification round-trip.
+ */
+ private List readGroupsWithSchema(java.nio.file.Path source, MessageType[] outSchema) throws IOException {
+ List rows = new ArrayList<>();
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(source))) {
+ MessageType schema = reader.getFooter().getFileMetaData().getSchema();
+ outSchema[0] = schema;
+ MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
+ PageReadStore pages;
+ while ((pages = reader.readNextRowGroup()) != null) {
+ long rowCount = pages.getRowCount();
+ RecordReader recordReader = columnIO.getRecordReader(pages, new GroupRecordConverter(schema));
+ for (long i = 0; i < rowCount; i++) {
+ rows.add(recordReader.read());
+ }
+ }
+ }
+ return rows;
+ }
+
+ /**
+ * Generates Java-written ALP fixtures from the four PR #100 source datasets across two axes:
+ * page version (V1, V2) and vectorSize (1024, 4096). Each output is verified bit-exact against
+ * its source. Produces 16 files total:
+ * {@code alp_java_{spotify1,arade,float_spotify1,float_arade}_v{1,2}_vs{1024,4096}.parquet}.
+ *
+ * Page version is orthogonal to ALP encoding (the page version difference lives in the
+ * parquet protocol layer, not in the ALP payload), but covering both axes makes the fixture
+ * set fully symmetric for cross-language compatibility — every reader can verify it handles
+ * Java-written ALP regardless of how the surrounding pages are framed.
+ *
+ *
To run: clone https://github.com/prtkgaur/parquet-testing/tree/alpFloatingPointDataset
+ * and point ALP_TEST_DATA_DIR at its {@code data/} directory. Set ALP_OUTPUT_DIR to choose
+ * where to write the generated fixtures; if unset, this test skips.
+ */
+ @Test
+ public void generateAlpFixturesAtMultipleVectorSizes() throws IOException {
+ java.nio.file.Path sourceDir = getTestDataDir();
+ assumeTrue("ALP source dir not found, skipping fixture generator", sourceDir != null);
+
+ java.nio.file.Path outDir = getOutputDir();
+ assumeTrue("ALP_OUTPUT_DIR not set, skipping fixture generator", outDir != null);
+ LOG.info("Generating ALP fixtures to {}", outDir);
+
+ int expectedFiles = SOURCE_FILES.length * GENERATOR_PAGE_VERSIONS.length * GENERATOR_VECTOR_SIZES.length;
+ int generated = 0;
+ for (String sourceFile : SOURCE_FILES) {
+ java.nio.file.Path source = sourceDir.resolve(sourceFile);
+ assumeTrue("Source missing: " + source, source.toFile().exists());
+
+ MessageType[] schemaHolder = new MessageType[1];
+ List sourceRows = readGroupsWithSchema(source, schemaHolder);
+ MessageType schema = schemaHolder[0];
+
+ String stem = sourceFile.replace("alp_", "").replace(".parquet", "");
+
+ for (WriterVersion pageVersion : GENERATOR_PAGE_VERSIONS) {
+ String pageTag = (pageVersion == WriterVersion.PARQUET_1_0) ? "v1" : "v2";
+ for (int vectorSize : GENERATOR_VECTOR_SIZES) {
+ java.nio.file.Path outPath =
+ outDir.resolve("alp_java_" + stem + "_" + pageTag + "_vs" + vectorSize + ".parquet");
+ java.nio.file.Files.deleteIfExists(outPath);
+
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(pageVersion)
+ .withAlpEncoding(true)
+ .withAlpVectorSize(vectorSize)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (Group row : sourceRows) {
+ writer.write(row);
+ }
+ }
+
+ // Verify: read back and compare against the source row-by-row
+ List roundTrip = readGroupsWithSchema(outPath, new MessageType[1]);
+ assertEquals(
+ "Row count mismatch for " + outPath.getFileName(), sourceRows.size(), roundTrip.size());
+ int fieldCount = schema.getFieldCount();
+ for (int i = 0; i < sourceRows.size(); i++) {
+ for (int f = 0; f < fieldCount; f++) {
+ PrimitiveType.PrimitiveTypeName type =
+ schema.getType(f).asPrimitiveType().getPrimitiveTypeName();
+ String fieldName = schema.getType(f).getName();
+ if (type == PrimitiveType.PrimitiveTypeName.DOUBLE) {
+ double expected = sourceRows.get(i).getDouble(f, 0);
+ double actual = roundTrip.get(i).getDouble(f, 0);
+ long eb = Double.doubleToRawLongBits(expected);
+ long ab = Double.doubleToRawLongBits(actual);
+ assertEquals(
+ "double bit mismatch in " + outPath.getFileName() + " row " + i + " field "
+ + fieldName,
+ eb,
+ ab);
+ } else if (type == PrimitiveType.PrimitiveTypeName.FLOAT) {
+ float expected = sourceRows.get(i).getFloat(f, 0);
+ float actual = roundTrip.get(i).getFloat(f, 0);
+ int eb = Float.floatToRawIntBits(expected);
+ int ab = Float.floatToRawIntBits(actual);
+ assertEquals(
+ "float bit mismatch in " + outPath.getFileName() + " row " + i + " field "
+ + fieldName,
+ eb,
+ ab);
+ }
+ }
+ }
+ long bytes = outPath.toFile().length();
+ LOG.info(
+ " wrote {} ({} rows, {} bytes, pageVersion={}, vectorSize={})",
+ outPath.getFileName(),
+ sourceRows.size(),
+ bytes,
+ pageTag,
+ vectorSize);
+ generated++;
+ }
+ }
+ }
+ assertEquals("Expected to generate " + expectedFiles + " fixture files", expectedFiles, generated);
+ LOG.info("Generated {} ALP fixture files to {}", generated, outDir);
+ }
+
+ // ---------------------------------------------------------------------------
+ // Reader-only validation: opens each Java-written fixture and asserts the
+ // reader path works end-to-end (metadata declares ALP, every row decodes
+ // without error). Separate from the generator's own round-trip check so it
+ // surfaces as a distinct signal — useful when the fixtures already exist on
+ // disk and you want to validate just the read path.
+ // ---------------------------------------------------------------------------
+
+ @Test
+ public void readAllFixtureFilesIndependently() throws IOException {
+ java.nio.file.Path outDir = getOutputDir();
+ assumeTrue("ALP_OUTPUT_DIR not set, skipping reader-only test", outDir != null);
+ File[] files = outDir.toFile().listFiles((f, n) -> n.startsWith("alp_java_") && n.endsWith(".parquet"));
+ assumeTrue("No fixture files in " + outDir, files != null && files.length > 0);
+ java.util.Arrays.sort(files, (a, b) -> a.getName().compareTo(b.getName()));
+
+ int filesRead = 0;
+ int totalChunks = 0;
+ int alpChunks = 0;
+ long totalRows = 0;
+ for (File pf : files) {
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(pf.toPath()))) {
+ ParquetMetadata footer = reader.getFooter();
+ // Layer 1: every column chunk declares ALP
+ for (org.apache.parquet.hadoop.metadata.BlockMetaData block : footer.getBlocks()) {
+ for (org.apache.parquet.hadoop.metadata.ColumnChunkMetaData col : block.getColumns()) {
+ totalChunks++;
+ if (col.getEncodings().contains(Encoding.ALP)) alpChunks++;
+ }
+ }
+ // Layer 2: every row actually decodes (no exceptions thrown, count matches footer)
+ MessageType schema = footer.getFileMetaData().getSchema();
+ MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
+ long rowsDecoded = 0;
+ PageReadStore pages;
+ while ((pages = reader.readNextRowGroup()) != null) {
+ long rowCount = pages.getRowCount();
+ RecordReader recordReader =
+ columnIO.getRecordReader(pages, new GroupRecordConverter(schema));
+ for (long i = 0; i < rowCount; i++) {
+ Group row = recordReader.read();
+ // Touch every present field so any decode-time exception surfaces.
+ // Optional/null fields have repetitionCount=0 — skip those rather than throw.
+ for (int f = 0; f < schema.getFieldCount(); f++) {
+ if (row.getFieldRepetitionCount(f) == 0) continue;
+ PrimitiveType.PrimitiveTypeName type =
+ schema.getType(f).asPrimitiveType().getPrimitiveTypeName();
+ if (type == PrimitiveType.PrimitiveTypeName.DOUBLE) {
+ row.getDouble(f, 0);
+ } else if (type == PrimitiveType.PrimitiveTypeName.FLOAT) {
+ row.getFloat(f, 0);
+ }
+ }
+ rowsDecoded++;
+ }
+ }
+ totalRows += rowsDecoded;
+ filesRead++;
+ }
+ }
+ assertEquals("Every column chunk should declare ALP", totalChunks, alpChunks);
+ LOG.info(
+ "readAllFixtureFilesIndependently: {} files, {} chunks ({} ALP), {} rows decoded",
+ filesRead,
+ totalChunks,
+ alpChunks,
+ totalRows);
+ }
+
+ // ---------------------------------------------------------------------------
+ // Corner-case fixture per parquet-testing issue #105. Writes a single small
+ // file with one ALP-encoded column per corner case (no exceptions, 1 exception,
+ // all exceptions, NaN/Inf/-0.0, nulls, constant/bit_width=0, multi-vector with
+ // differing exponents) for both f32 and f64, then reads back and verifies
+ // every value bit-exactly against the expected pattern.
+ //
+ // The output file (alp_java_cornercases.parquet) is intended as a candidate
+ // fixture for parquet-testing PR #100 once Prateek confirms naming and design.
+ // ---------------------------------------------------------------------------
+
+ private static final int CORNER_ROWS_PER_VECTOR = 1024;
+ private static final int CORNER_VECTORS = 2;
+ private static final int CORNER_TOTAL_ROWS = CORNER_ROWS_PER_VECTOR * CORNER_VECTORS;
+
+ /** Expected values per corner-case column. Doubles use NaN to mark a null cell for sentinel use. */
+ private static class CornerCaseData {
+ double[] doubleVals;
+ float[] floatVals;
+ boolean[] isNull; // only meaningful for optional columns
+ boolean isFloat;
+ boolean isOptional;
+ String name;
+
+ static CornerCaseData doubles(String name, double[] vals) {
+ CornerCaseData d = new CornerCaseData();
+ d.name = name;
+ d.doubleVals = vals;
+ d.isFloat = false;
+ d.isOptional = false;
+ return d;
+ }
+
+ static CornerCaseData floats(String name, float[] vals) {
+ CornerCaseData d = new CornerCaseData();
+ d.name = name;
+ d.floatVals = vals;
+ d.isFloat = true;
+ d.isOptional = false;
+ return d;
+ }
+
+ static CornerCaseData optionalDoubles(String name, double[] vals, boolean[] nulls) {
+ CornerCaseData d = doubles(name, vals);
+ d.isOptional = true;
+ d.isNull = nulls;
+ return d;
+ }
+
+ static CornerCaseData optionalFloats(String name, float[] vals, boolean[] nulls) {
+ CornerCaseData d = floats(name, vals);
+ d.isOptional = true;
+ d.isNull = nulls;
+ return d;
+ }
+ }
+
+ private List buildCornerCaseColumns() {
+ List cols = new ArrayList<>();
+ int N = CORNER_TOTAL_ROWS;
+
+ // 1. f64_no_exceptions: clean 2-decimal values, both vectors low bit-width
+ double[] noExc = new double[N];
+ for (int i = 0; i < N; i++) noExc[i] = (i % 1000) / 100.0;
+ cols.add(CornerCaseData.doubles("f64_no_exceptions", noExc));
+
+ // 2. f64_one_exception_per_vector: clean values with 1 NaN per 1024-row vector
+ double[] oneExc = new double[N];
+ for (int i = 0; i < N; i++) oneExc[i] = (i % 1000) / 100.0;
+ oneExc[100] = Double.NaN;
+ oneExc[1100] = Double.NaN;
+ cols.add(CornerCaseData.doubles("f64_one_exception_per_vector", oneExc));
+
+ // 3. f64_all_nan: every value NaN — all-exception vector
+ double[] allNan = new double[N];
+ java.util.Arrays.fill(allNan, Double.NaN);
+ cols.add(CornerCaseData.doubles("f64_all_nan", allNan));
+
+ // 4. f64_nan_inf_negzero: mix of special values across two vectors
+ double[] mixed = new double[N];
+ for (int i = 0; i < N; i++) mixed[i] = (i % 100) / 10.0;
+ mixed[0] = Double.NaN;
+ mixed[1] = Double.POSITIVE_INFINITY;
+ mixed[2] = Double.NEGATIVE_INFINITY;
+ mixed[3] = -0.0;
+ mixed[1024] = Double.NaN;
+ mixed[1025] = -0.0;
+ cols.add(CornerCaseData.doubles("f64_nan_inf_negzero", mixed));
+
+ // 5. f64_constant: every value = 42.0 → max_delta=0 → bit_width=0
+ double[] constant = new double[N];
+ java.util.Arrays.fill(constant, 42.0);
+ cols.add(CornerCaseData.doubles("f64_constant", constant));
+
+ // 6. f64_different_e_f_per_vector: vector 0 uses e=2,f=0 (cents); vector 1 uses e=3,f=0 (mils)
+ double[] diff = new double[N];
+ for (int i = 0; i < CORNER_ROWS_PER_VECTOR; i++) diff[i] = (i % 100) / 100.0;
+ for (int i = CORNER_ROWS_PER_VECTOR; i < N; i++) diff[i] = ((i - CORNER_ROWS_PER_VECTOR) % 1000) / 1000.0;
+ cols.add(CornerCaseData.doubles("f64_different_ef_per_vector", diff));
+
+ // 7. f64_with_nulls: optional column, ~30% nulls scattered
+ double[] withNulls = new double[N];
+ boolean[] nullMask = new boolean[N];
+ for (int i = 0; i < N; i++) {
+ if (i % 3 == 0) {
+ nullMask[i] = true;
+ } else {
+ withNulls[i] = (i % 100) / 100.0;
+ }
+ }
+ cols.add(CornerCaseData.optionalDoubles("f64_with_nulls", withNulls, nullMask));
+
+ // 8. f64_wide_for_range: integer-valued doubles spanning a wide signed range so the
+ // frame-of-reference minimum is deeply negative and the delta (max - min) needs a high
+ // bit width. Exercises the signed FOR-range computation (a buggy unsigned max - min would
+ // overstate the range here). Values stay below 2^52 so they encode exactly with no exceptions.
+ double[] wideD = new double[N];
+ final double loD = -4.5e15, hiD = 4.5e15;
+ for (int i = 0; i < N; i++) wideD[i] = (i % 2 == 0) ? loD + i : hiD - i;
+ wideD[0] = loD; // frame minimum near -2^52
+ wideD[1] = hiD; // frame maximum near +2^52
+ cols.add(CornerCaseData.doubles("f64_wide_for_range", wideD));
+
+ // f64_extreme_for_64bit: exact integer-valued doubles near the ALP encoding limit (~2^63) spanning
+ // the full range, so the FOR delta needs the full 64-bit width (the signed max - min subtraction
+ // overflows) with zero exceptions. This is the extreme-value case that f64_wide_for_range (~54 bits)
+ // does not reach; it stresses 64-bit FOR packing and the reader's modular reconstruction across
+ // implementations.
+ double[] extremeD = new double[N];
+ final long loLD = -9_000_000_000_000_000_000L, hiLD = 9_000_000_000_000_000_000L;
+ for (int i = 0; i < N; i++) {
+ long off = (long) (i % 256) * (1L << 20);
+ extremeD[i] = (double) ((i % 2 == 0) ? (loLD + off) : (hiLD - off));
+ }
+ cols.add(CornerCaseData.doubles("f64_extreme_for_64bit", extremeD));
+
+ // f64_extreme_for_63bit: exact integer-valued doubles near +/- 2^61, so the FOR delta needs 63 bits
+ // WITHOUT overflowing the signed max - min subtraction (the non-overflow high-bit counterpart to
+ // f64_extreme_for_64bit), with zero exceptions.
+ double[] extreme63D = new double[N];
+ final long lo63 = -3_000_000_000_000_000_000L, hi63 = 3_000_000_000_000_000_000L;
+ for (int i = 0; i < N; i++) {
+ long off = (long) (i % 256) * (1L << 20);
+ extreme63D[i] = (double) ((i % 2 == 0) ? (lo63 + off) : (hi63 - off));
+ }
+ cols.add(CornerCaseData.doubles("f64_extreme_for_63bit", extreme63D));
+
+ // 9–15: f32 analogues
+ float[] noExcF = new float[N];
+ for (int i = 0; i < N; i++) noExcF[i] = (i % 1000) / 100.0f;
+ cols.add(CornerCaseData.floats("f32_no_exceptions", noExcF));
+
+ float[] oneExcF = new float[N];
+ for (int i = 0; i < N; i++) oneExcF[i] = (i % 1000) / 100.0f;
+ oneExcF[100] = Float.NaN;
+ oneExcF[1100] = Float.NaN;
+ cols.add(CornerCaseData.floats("f32_one_exception_per_vector", oneExcF));
+
+ float[] allNanF = new float[N];
+ java.util.Arrays.fill(allNanF, Float.NaN);
+ cols.add(CornerCaseData.floats("f32_all_nan", allNanF));
+
+ float[] mixedF = new float[N];
+ for (int i = 0; i < N; i++) mixedF[i] = (i % 100) / 10.0f;
+ mixedF[0] = Float.NaN;
+ mixedF[1] = Float.POSITIVE_INFINITY;
+ mixedF[2] = Float.NEGATIVE_INFINITY;
+ mixedF[3] = -0.0f;
+ mixedF[1024] = Float.NaN;
+ mixedF[1025] = -0.0f;
+ cols.add(CornerCaseData.floats("f32_nan_inf_negzero", mixedF));
+
+ float[] constantF = new float[N];
+ java.util.Arrays.fill(constantF, 42.0f);
+ cols.add(CornerCaseData.floats("f32_constant", constantF));
+
+ float[] diffF = new float[N];
+ for (int i = 0; i < CORNER_ROWS_PER_VECTOR; i++) diffF[i] = (i % 100) / 100.0f;
+ for (int i = CORNER_ROWS_PER_VECTOR; i < N; i++) diffF[i] = ((i - CORNER_ROWS_PER_VECTOR) % 1000) / 1000.0f;
+ cols.add(CornerCaseData.floats("f32_different_ef_per_vector", diffF));
+
+ float[] withNullsF = new float[N];
+ boolean[] nullMaskF = new boolean[N];
+ for (int i = 0; i < N; i++) {
+ if (i % 3 == 0) {
+ nullMaskF[i] = true;
+ } else {
+ withNullsF[i] = (i % 100) / 100.0f;
+ }
+ }
+ cols.add(CornerCaseData.optionalFloats("f32_with_nulls", withNullsF, nullMaskF));
+
+ // f32_wide_for_range: integer-valued floats spanning the widest signed range float can hold
+ // losslessly (< 2^23), so the frame-of-reference minimum is deeply negative and the FOR delta
+ // needs a high bit width. Mirrors f64_wide_for_range for the int-encoded (float) path.
+ float[] wideF = new float[N];
+ final float loF = -8.3e6f, hiF = 8.3e6f;
+ for (int i = 0; i < N; i++) wideF[i] = (i % 2 == 0) ? loF + i : hiF - i;
+ wideF[0] = loF; // frame minimum near -2^23
+ wideF[1] = hiF; // frame maximum near +2^23
+ cols.add(CornerCaseData.floats("f32_wide_for_range", wideF));
+
+ // f32_extreme_for_32bit: exact integer-valued floats near the ~2^31 float encoding limit spanning
+ // the full range, forcing the full 32-bit FOR width with zero exceptions (the int-encoded analogue
+ // of f64_extreme_for_64bit).
+ float[] extremeF = new float[N];
+ final int loIF = -2_100_000_000, hiIF = 2_100_000_000;
+ for (int i = 0; i < N; i++) {
+ int off = (i % 256) * (1 << 8);
+ extremeF[i] = (float) ((i % 2 == 0) ? (loIF + off) : (hiIF - off));
+ }
+ cols.add(CornerCaseData.floats("f32_extreme_for_32bit", extremeF));
+
+ return cols;
+ }
+
+ /**
+ * Dumps the corner-case construction recipe to a CSV truth file alongside the parquet.
+ * The CSV is built directly from the expected values (not by reading the parquet back),
+ * which makes it suitable as independent ground truth for cross-language verifiers.
+ *
+ * Conventions:
+ *
+ * - Header row matches column names.
+ *
- Empty field marks a null cell in an optional column.
+ *
- Special values use Java's standard toString output: {@code NaN}, {@code Infinity},
+ * {@code -Infinity}. These parse correctly via C++ {@code std::stod} / {@code std::stof}
+ * (per the standard, both are case-insensitive and accept the full word "Infinity").
+ *
- {@code -0.0} round-trips through both Java and C++ string conversion bit-exact.
+ *
+ */
+ private void writeCornerCaseCsvTruth(java.nio.file.Path csvPath, List cols) throws IOException {
+ try (java.io.BufferedWriter w = java.nio.file.Files.newBufferedWriter(csvPath)) {
+ // Header
+ for (int i = 0; i < cols.size(); i++) {
+ if (i > 0) w.write(",");
+ w.write(cols.get(i).name);
+ }
+ w.write("\n");
+ // Data
+ for (int r = 0; r < CORNER_TOTAL_ROWS; r++) {
+ for (int i = 0; i < cols.size(); i++) {
+ if (i > 0) w.write(",");
+ CornerCaseData c = cols.get(i);
+ if (c.isOptional && c.isNull[r]) {
+ // empty field = null
+ } else if (c.isFloat) {
+ w.write(Float.toString(c.floatVals[r]));
+ } else {
+ w.write(Double.toString(c.doubleVals[r]));
+ }
+ }
+ w.write("\n");
+ }
+ }
+ }
+
+ @Test
+ public void generateAndVerifyCornerCaseFixture() throws IOException {
+ java.nio.file.Path outDir = getOutputDir();
+ assumeTrue("ALP_OUTPUT_DIR not set, skipping corner-case generator", outDir != null);
+ java.nio.file.Path outPath = outDir.resolve("alp_java_cornercases.parquet");
+ java.nio.file.Path csvPath = outDir.resolve("alp_java_cornercases_expect.csv");
+ java.nio.file.Files.deleteIfExists(outPath);
+ java.nio.file.Files.deleteIfExists(csvPath);
+
+ List cols = buildCornerCaseColumns();
+ writeCornerCaseCsvTruth(csvPath, cols);
+
+ // Build the schema
+ StringBuilder schemaText = new StringBuilder("message alp_corner_cases {\n");
+ for (CornerCaseData c : cols) {
+ schemaText
+ .append(" ")
+ .append(c.isOptional ? "optional " : "required ")
+ .append(c.isFloat ? "float " : "double ")
+ .append(c.name)
+ .append(";\n");
+ }
+ schemaText.append("}\n");
+ MessageType schema = MessageTypeParser.parseMessageType(schemaText.toString());
+
+ // Write
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withAlpVectorSize(CORNER_ROWS_PER_VECTOR)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int r = 0; r < CORNER_TOTAL_ROWS; r++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ for (CornerCaseData c : cols) {
+ if (c.isOptional && c.isNull[r]) continue; // leave field unset
+ if (c.isFloat) {
+ row.add(c.name, c.floatVals[r]);
+ } else {
+ row.add(c.name, c.doubleVals[r]);
+ }
+ }
+ writer.write(row);
+ }
+ }
+
+ // Read back and verify every value bit-exact
+ int mismatches = 0;
+ int values = 0;
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(outPath))) {
+ MessageType readSchema = reader.getFooter().getFileMetaData().getSchema();
+ MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(readSchema);
+ PageReadStore pages;
+ int rowIdx = 0;
+ while ((pages = reader.readNextRowGroup()) != null) {
+ long rc = pages.getRowCount();
+ RecordReader rr = columnIO.getRecordReader(pages, new GroupRecordConverter(readSchema));
+ for (long i = 0; i < rc; i++) {
+ Group row = rr.read();
+ for (int fi = 0; fi < cols.size(); fi++) {
+ CornerCaseData c = cols.get(fi);
+ boolean shouldBePresent = !(c.isOptional && c.isNull[rowIdx]);
+ int fieldRepetition = row.getFieldRepetitionCount(fi);
+ if (!shouldBePresent) {
+ if (fieldRepetition != 0) {
+ mismatches++;
+ LOG.warn("col={} row={} expected null but got value", c.name, rowIdx);
+ }
+ continue;
+ }
+ if (fieldRepetition == 0) {
+ mismatches++;
+ LOG.warn("col={} row={} expected value but got null", c.name, rowIdx);
+ continue;
+ }
+ values++;
+ if (c.isFloat) {
+ float expected = c.floatVals[rowIdx];
+ float actual = row.getFloat(fi, 0);
+ if (Float.floatToRawIntBits(expected) != Float.floatToRawIntBits(actual)) {
+ if (mismatches < 5) {
+ LOG.warn("col={} row={} expected={} actual={}", c.name, rowIdx, expected, actual);
+ }
+ mismatches++;
+ }
+ } else {
+ double expected = c.doubleVals[rowIdx];
+ double actual = row.getDouble(fi, 0);
+ if (Double.doubleToRawLongBits(expected) != Double.doubleToRawLongBits(actual)) {
+ if (mismatches < 5) {
+ LOG.warn("col={} row={} expected={} actual={}", c.name, rowIdx, expected, actual);
+ }
+ mismatches++;
+ }
+ }
+ }
+ rowIdx++;
+ }
+ }
+ }
+ assertEquals("Corner-case fixture had decode mismatches", 0, mismatches);
+ LOG.info(
+ "generateAndVerifyCornerCaseFixture: wrote {} ({} bytes) and {} ({} bytes); {} cols, {} rows, {} values verified",
+ outPath.getFileName(),
+ outPath.toFile().length(),
+ csvPath.getFileName(),
+ csvPath.toFile().length(),
+ cols.size(),
+ CORNER_TOTAL_ROWS,
+ values);
+ }
+
+ // ---------------------------------------------------------------------------
+ // Statistics correctness for ALP-encoded columns.
+ //
+ // Statistics are populated by the column writer wrapper (not the encoder),
+ // so they should "just work" regardless of which encoding is in use — but
+ // the assumption is worth pinning. Wrong/missing statistics break parquet
+ // predicate pushdown: readers either skip row groups they shouldn't or scan
+ // ones they could have skipped.
+ // ---------------------------------------------------------------------------
+
+ @Test
+ public void testAlpColumnStatisticsAreCorrect() throws IOException {
+ MessageType schema = MessageTypeParser.parseMessageType(ALP_SCHEMA);
+ // Mix positive / negative / zero so min/max aren't trivially the endpoints
+ double[] doubleVals = {1.23, -4.56, 7.89, 0.0, 1000.0, -3.14, 9.99, 0.001};
+ float[] floatVals = {1.23f, -4.56f, 7.89f, 0.0f, 1000.0f, -3.14f, 9.99f, 0.001f};
+ double expectedDoubleMin = -4.56, expectedDoubleMax = 1000.0;
+ float expectedFloatMin = -4.56f, expectedFloatMax = 1000.0f;
+
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_stats.parquet");
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < doubleVals.length; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("double_col", doubleVals[i]);
+ row.add("float_col", floatVals[i]);
+ writer.write(row);
+ }
+ }
+
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(outPath))) {
+ ParquetMetadata footer = reader.getFooter();
+ assertEquals("expected one row group", 1, footer.getBlocks().size());
+ org.apache.parquet.hadoop.metadata.BlockMetaData block =
+ footer.getBlocks().get(0);
+
+ org.apache.parquet.hadoop.metadata.ColumnChunkMetaData doubleChunk = null;
+ org.apache.parquet.hadoop.metadata.ColumnChunkMetaData floatChunk = null;
+ for (org.apache.parquet.hadoop.metadata.ColumnChunkMetaData c : block.getColumns()) {
+ String path = c.getPath().toDotString();
+ if (path.equals("double_col")) doubleChunk = c;
+ if (path.equals("float_col")) floatChunk = c;
+ }
+ assertTrue(
+ "double_col must be ALP-encoded", doubleChunk.getEncodings().contains(Encoding.ALP));
+ assertTrue(
+ "float_col must be ALP-encoded", floatChunk.getEncodings().contains(Encoding.ALP));
+
+ org.apache.parquet.column.statistics.Statistics dStats = doubleChunk.getStatistics();
+ org.apache.parquet.column.statistics.Statistics fStats = floatChunk.getStatistics();
+
+ assertTrue("double stats must have min/max set", dStats.hasNonNullValue());
+ assertTrue("float stats must have min/max set", fStats.hasNonNullValue());
+
+ // Bit-exact: same encoder path produces same IEEE 754 representation;
+ // statistics min/max should reflect the input data exactly.
+ assertEquals(
+ "double min mismatch",
+ Double.doubleToRawLongBits(expectedDoubleMin),
+ Double.doubleToRawLongBits((Double) dStats.genericGetMin()));
+ assertEquals(
+ "double max mismatch",
+ Double.doubleToRawLongBits(expectedDoubleMax),
+ Double.doubleToRawLongBits((Double) dStats.genericGetMax()));
+ assertEquals(
+ "float min mismatch", Float.floatToRawIntBits(expectedFloatMin), Float.floatToRawIntBits((Float)
+ fStats.genericGetMin()));
+ assertEquals(
+ "float max mismatch", Float.floatToRawIntBits(expectedFloatMax), Float.floatToRawIntBits((Float)
+ fStats.genericGetMax()));
+ assertEquals("double null count must be 0", 0L, dStats.getNumNulls());
+ assertEquals("float null count must be 0", 0L, fStats.getNumNulls());
+ }
+ }
+
+ // ---------------------------------------------------------------------------
+ // Integration coverage: the production paths normal ALP tests skip (dictionary
+ // fallback, compression, adversarial statistics, repeated/nested columns).
+ // ---------------------------------------------------------------------------
+
+ private boolean columnUsesAlp(java.nio.file.Path file, String colDotPath) throws IOException {
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(file))) {
+ for (org.apache.parquet.hadoop.metadata.BlockMetaData b :
+ reader.getFooter().getBlocks()) {
+ for (org.apache.parquet.hadoop.metadata.ColumnChunkMetaData c : b.getColumns()) {
+ if ((colDotPath == null || c.getPath().toDotString().equals(colDotPath))
+ && c.getEncodings().contains(Encoding.ALP)) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void testDictionaryOverflowFallsBackToAlp() throws IOException {
+ // Dictionary ENABLED (the real default) with a small dictionary page, so a high-cardinality
+ // double/float column overflows the dictionary and FALLS BACK to ALP mid-column. Normal ALP
+ // tests disable the dictionary, so this exercises the untested production path where the
+ // FallbackValuesWriter replays buffered values into ALP.
+ MessageType schema = MessageTypeParser.parseMessageType("message m { required double d; required float f; }");
+ int n = 20000;
+ double[] ds = new double[n];
+ float[] fs = new float[n];
+ java.util.Random rng = new java.util.Random(3);
+ for (int i = 0; i < n; i++) {
+ ds[i] = Math.round(rng.nextDouble() * 1e9) / 100.0;
+ fs[i] = (float) (Math.round(rng.nextFloat() * 1e6) / 100.0);
+ }
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_dict_fallback.parquet");
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(true)
+ .withDictionaryPageSize(4096)
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < n; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("d", ds[i]);
+ row.add("f", fs[i]);
+ writer.write(row);
+ }
+ }
+ List rows = readAllRows(outPath);
+ assertEquals(n, rows.size());
+ for (int i = 0; i < n; i++) {
+ assertEquals(
+ "d mismatch at " + i,
+ Double.doubleToRawLongBits(ds[i]),
+ Double.doubleToRawLongBits(rows.get(i).getDouble("d", 0)));
+ assertEquals(
+ "f mismatch at " + i,
+ Float.floatToRawIntBits(fs[i]),
+ Float.floatToRawIntBits(rows.get(i).getFloat("f", 0)));
+ }
+ assertTrue("Expected dictionary overflow to fall back to ALP encoding", columnUsesAlp(outPath, null));
+ }
+
+ @Test
+ public void testAlpUnderCompressionCodecs() throws IOException {
+ MessageType schema = MessageTypeParser.parseMessageType("message m { required double d; required float f; }");
+ int n = 5000;
+ double[] ds = new double[n];
+ float[] fs = new float[n];
+ for (int i = 0; i < n; i++) {
+ ds[i] = (i % 1000) / 100.0 - 5.0;
+ fs[i] = (i % 777) / 100.0f - 3.0f;
+ }
+ for (CompressionCodecName codec : new CompressionCodecName[] {
+ CompressionCodecName.SNAPPY, CompressionCodecName.GZIP, CompressionCodecName.ZSTD
+ }) {
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_" + codec.name() + ".parquet");
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(codec)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < n; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("d", ds[i]);
+ row.add("f", fs[i]);
+ writer.write(row);
+ }
+ }
+ List rows = readAllRows(outPath);
+ assertEquals("row count for " + codec, n, rows.size());
+ for (int i = 0; i < n; i++) {
+ assertEquals(
+ "d mismatch " + codec + " row " + i,
+ Double.doubleToRawLongBits(ds[i]),
+ Double.doubleToRawLongBits(rows.get(i).getDouble("d", 0)));
+ assertEquals(
+ "f mismatch " + codec + " row " + i,
+ Float.floatToRawIntBits(fs[i]),
+ Float.floatToRawIntBits(rows.get(i).getFloat("f", 0)));
+ }
+ assertTrue(codec + ": column should be ALP-encoded", columnUsesAlp(outPath, "d"));
+ }
+ }
+
+ @Test
+ public void testAlpStatisticsExcludeNaNAndCountNulls() throws IOException {
+ // Optional double column mixing finite values, NaN (must be excluded from min/max), and nulls
+ // (must be counted). Combines def-level nulls (num_elements < rowCount) with ALP exceptions (NaN)
+ // and verifies the footer statistics that predicate pushdown relies on are correct.
+ MessageType schema = MessageTypeParser.parseMessageType("message m { optional double d; }");
+ Double[] vals = {1.0, Double.NaN, -2.0, 5.0, null, 3.0, Double.NaN, null, 0.0};
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_stats_adv.parquet");
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (Double v : vals) {
+ SimpleGroup row = new SimpleGroup(schema);
+ if (v != null) {
+ row.add("d", v.doubleValue());
+ }
+ writer.write(row);
+ }
+ }
+ // Values also round-trip (with NaN payload preserved and nulls in the right places).
+ List rows = readAllRows(outPath);
+ assertEquals(vals.length, rows.size());
+ for (int i = 0; i < vals.length; i++) {
+ int rep = rows.get(i).getFieldRepetitionCount("d");
+ if (vals[i] == null) {
+ assertEquals("row " + i + " should be null", 0, rep);
+ } else {
+ assertEquals("row " + i + " should be present", 1, rep);
+ assertEquals(
+ "value row " + i,
+ Double.doubleToRawLongBits(vals[i]),
+ Double.doubleToRawLongBits(rows.get(i).getDouble("d", 0)));
+ }
+ }
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(outPath))) {
+ org.apache.parquet.hadoop.metadata.ColumnChunkMetaData chunk =
+ reader.getFooter().getBlocks().get(0).getColumns().get(0);
+ org.apache.parquet.column.statistics.Statistics s = chunk.getStatistics();
+ assertEquals("null count", 2L, s.getNumNulls());
+ double min = (Double) s.genericGetMin();
+ double max = (Double) s.genericGetMax();
+ assertTrue("min must not be NaN", !Double.isNaN(min));
+ assertTrue("max must not be NaN", !Double.isNaN(max));
+ assertEquals("min", -2.0, min, 0.0);
+ assertEquals("max", 5.0, max, 0.0);
+ }
+ }
+
+ @Test
+ public void testAlpLargeScaleManyRowGroups() throws IOException {
+ // Half a million rows with a small row-group size, forcing many row groups, exercised through
+ // the full write/read pipeline (compression on). Verification is streaming (regenerate the
+ // expected sequence, no List accumulation) so the read side is memory-safe at scale too.
+ MessageType schema = MessageTypeParser.parseMessageType("message m { required double d; required float f; }");
+ int n = 500_000;
+ long seed = 5;
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_large.parquet");
+ java.util.Random rng = new java.util.Random(seed);
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.SNAPPY)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withRowGroupSize(1L << 19) // 512KB, so 500k rows span many row groups
+ .withConf(new Configuration())
+ .build()) {
+ for (int i = 0; i < n; i++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("d", Math.round(rng.nextDouble() * 1e7) / 100.0);
+ row.add("f", (float) (Math.round(rng.nextFloat() * 1e5) / 100.0));
+ writer.write(row);
+ }
+ }
+ java.util.Random verify = new java.util.Random(seed);
+ int total = 0;
+ int rowGroups = 0;
+ try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(outPath))) {
+ MessageType s = reader.getFooter().getFileMetaData().getSchema();
+ MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(s);
+ PageReadStore pages;
+ while ((pages = reader.readNextRowGroup()) != null) {
+ rowGroups++;
+ long rc = pages.getRowCount();
+ RecordReader rr = columnIO.getRecordReader(pages, new GroupRecordConverter(s));
+ for (long i = 0; i < rc; i++) {
+ Group g = rr.read();
+ double ed = Math.round(verify.nextDouble() * 1e7) / 100.0;
+ float ef = (float) (Math.round(verify.nextFloat() * 1e5) / 100.0);
+ assertEquals(
+ "d mismatch at row " + total,
+ Double.doubleToRawLongBits(ed),
+ Double.doubleToRawLongBits(g.getDouble("d", 0)));
+ assertEquals(
+ "f mismatch at row " + total,
+ Float.floatToRawIntBits(ef),
+ Float.floatToRawIntBits(g.getFloat("f", 0)));
+ total++;
+ }
+ }
+ }
+ assertEquals("row count", n, total);
+ assertTrue("expected multiple row groups, got " + rowGroups, rowGroups >= 2);
+ }
+
+ @Test
+ public void testAlpOnRepeatedDoubleField() throws IOException {
+ // ALP on a REPEATED double field, so it must interleave correctly with repetition/definition
+ // levels (empty rows, varying counts). Nested/repeated columns are otherwise untested.
+ MessageType schema =
+ MessageTypeParser.parseMessageType("message m { required int64 id; repeated double vals; }");
+ int nRows = 3000;
+ java.util.Random rng = new java.util.Random(9);
+ double[][] expected = new double[nRows][];
+ for (int r = 0; r < nRows; r++) {
+ int k = rng.nextInt(5); // 0..4 values per row (exercises empty rows + varying rep levels)
+ expected[r] = new double[k];
+ for (int j = 0; j < k; j++) {
+ expected[r][j] = Math.round(rng.nextDouble() * 100000) / 100.0;
+ }
+ }
+ java.nio.file.Path outPath = temp.newFolder().toPath().resolve("alp_repeated.parquet");
+ try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(outPath))
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withWriterVersion(WriterVersion.PARQUET_2_0)
+ .withAlpEncoding(true)
+ .withDictionaryEncoding(false)
+ .withConf(new Configuration())
+ .build()) {
+ for (int r = 0; r < nRows; r++) {
+ SimpleGroup row = new SimpleGroup(schema);
+ row.add("id", (long) r);
+ for (double v : expected[r]) {
+ row.add("vals", v);
+ }
+ writer.write(row);
+ }
+ }
+ List rows = readAllRows(outPath);
+ assertEquals(nRows, rows.size());
+ for (int r = 0; r < nRows; r++) {
+ Group g = rows.get(r);
+ int k = g.getFieldRepetitionCount("vals");
+ assertEquals("rep count row " + r, expected[r].length, k);
+ for (int j = 0; j < k; j++) {
+ assertEquals(
+ "val row " + r + " idx " + j,
+ Double.doubleToRawLongBits(expected[r][j]),
+ Double.doubleToRawLongBits(g.getDouble("vals", j)));
+ }
+ }
+ assertTrue("repeated double column should be ALP-encoded", columnUsesAlp(outPath, "vals"));
+ }
+}