Skip to content

Commit 91eebbf

Browse files
NicolaeNMVclaude
andcommitted
GH-3967: Skip the page copy for decompressors with stable output
GH-3150 (#3801) copies every decompressed page in ColumnChunkPageReadStore, because the BytesInputDecompressor contract allows a decompressor to reuse its output buffer. On the default heap path that copy is redundant for UNCOMPRESSED (NO_OP returns the page as read) and for ZSTD/LZ4_RAW (HeapBytesDecompressor already decompresses into a fresh array). Add a package-private CodecFactory.BytesDecompressor.returnsStableBytes(), false by default and overridden by NO_OP and by HeapBytesDecompressor for eagerly materialized codecs. The reader skips the copy only for those; DirectCodecFactory and third-party decompressors are still copied. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 567c0ec commit 91eebbf

3 files changed

Lines changed: 125 additions & 7 deletions

File tree

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/CodecFactory.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,11 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) {
8888
return bytes;
8989
}
9090

91+
@Override
92+
boolean returnsStableBytes() {
93+
return true; // returns the input, which the caller owns
94+
}
95+
9196
@Override
9297
public void release() {}
9398
};
@@ -173,6 +178,15 @@ class HeapBytesDecompressor extends BytesDecompressor {
173178
decompressor = CodecPool.getDecompressor(codec);
174179
}
175180

181+
private boolean materializesEagerly() {
182+
return codec instanceof ZstandardCodec || codec instanceof Lz4RawCodec;
183+
}
184+
185+
@Override
186+
boolean returnsStableBytes() {
187+
return materializesEagerly(); // a fresh array; the lazy stream of other codecs is tied to the decompressor
188+
}
189+
176190
@Override
177191
public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOException {
178192
final BytesInput decompressed;
@@ -192,7 +206,7 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOEx
192206
// LZ4_RAW: requires one-shot decompression; the lazy StreamBytesInput.writeInto() path reads via
193207
// Channels.newChannel() in ~8KB chunks, causing the decompressor to be called with an undersized
194208
// output buffer (see #3478).
195-
if (codec instanceof ZstandardCodec || codec instanceof Lz4RawCodec) {
209+
if (materializesEagerly()) {
196210
decompressed = BytesInput.copy(BytesInput.from(is, decompressedSize));
197211
is.close();
198212
} else {
@@ -514,5 +528,14 @@ public abstract void decompress(ByteBuffer input, int compressedSize, ByteBuffer
514528
throws IOException;
515529

516530
public abstract void release();
531+
532+
/**
533+
* Whether the {@link BytesInput} returned by {@link #decompress(BytesInput, int)} stays valid when this
534+
* decompressor is used again, so callers need not copy it. Package-private: only implementations in this
535+
* package can opt in; any other implementation is assumed to reuse its buffers.
536+
*/
537+
boolean returnsStableBytes() {
538+
return false;
539+
}
517540
}
518541
}

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,7 @@ public DataPage visit(DataPageV1 dataPageV1) {
177177
bytes = BytesInput.from(blockDecryptor.decrypt(bytes.toByteArray(), dataPageAAD));
178178
}
179179
long start = System.nanoTime();
180-
decompressed =
181-
BytesInput.copy(decompressor.decompress(bytes, dataPageV1.getUncompressedSize()));
180+
decompressed = ownedCopy(decompressor.decompress(bytes, dataPageV1.getUncompressedSize()));
182181
setDecompressMetrics(bytes, start);
183182
}
184183

@@ -258,7 +257,7 @@ public DataPage visit(DataPageV2 dataPageV2) {
258257
- dataPageV2.getDefinitionLevels().size()
259258
- dataPageV2.getRepetitionLevels().size());
260259
long start = System.nanoTime();
261-
pageBytes = BytesInput.copy(decompressor.decompress(pageBytes, uncompressedSize));
260+
pageBytes = ownedCopy(decompressor.decompress(pageBytes, uncompressedSize));
262261
setDecompressMetrics(pageBytes, start);
263262
}
264263
}
@@ -297,6 +296,24 @@ public DataPage visit(DataPageV2 dataPageV2) {
297296
});
298297
}
299298

299+
/**
300+
* The {@link BytesInputDecompressor} contract allows the returned bytes to be overwritten by the next call, but
301+
* pages are kept after the decompressor moves on (GH-3150), so they are copied unless the decompressor guarantees
302+
* stable output.
303+
*/
304+
private BytesInput ownedCopy(BytesInput decompressed) throws IOException {
305+
return returnsStableBytes() ? decompressed : BytesInput.copy(decompressed);
306+
}
307+
308+
private BytesInput ownedCopy(BytesInput decompressed, ByteBufferReleaser releaser) throws IOException {
309+
return returnsStableBytes() ? decompressed : decompressed.copy(releaser);
310+
}
311+
312+
private boolean returnsStableBytes() {
313+
return decompressor instanceof CodecFactory.BytesDecompressor
314+
&& ((CodecFactory.BytesDecompressor) decompressor).returnsStableBytes();
315+
}
316+
300317
private void setDecompressMetrics(BytesInput bytes, long start) {
301318
final ParquetMetricsCallback metricsCallback = options.getMetricsCallback();
302319
if (metricsCallback != null) {
@@ -327,9 +344,9 @@ public DictionaryPage readDictionaryPage() {
327344
long start = System.nanoTime();
328345
setDecompressMetrics(bytes, start);
329346
DictionaryPage decompressedPage = new DictionaryPage(
330-
decompressor
331-
.decompress(bytes, compressedDictionaryPage.getUncompressedSize())
332-
.copy(releaser),
347+
ownedCopy(
348+
decompressor.decompress(bytes, compressedDictionaryPage.getUncompressedSize()),
349+
releaser),
333350
compressedDictionaryPage.getDictionarySize(),
334351
compressedDictionaryPage.getEncoding());
335352
if (compressedDictionaryPage.getCrc().isPresent()) {

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageReadStore.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,21 @@
2929

3030
import java.nio.ByteBuffer;
3131
import java.util.Collections;
32+
import org.apache.hadoop.conf.Configuration;
3233
import org.apache.parquet.ParquetReadOptions;
3334
import org.apache.parquet.bytes.ByteBufferAllocator;
3435
import org.apache.parquet.bytes.ByteBufferReleaser;
3536
import org.apache.parquet.bytes.BytesInput;
37+
import org.apache.parquet.bytes.DirectByteBufferAllocator;
3638
import org.apache.parquet.bytes.HeapByteBufferAllocator;
3739
import org.apache.parquet.bytes.TrackingByteBufferAllocator;
3840
import org.apache.parquet.column.ColumnDescriptor;
3941
import org.apache.parquet.column.page.DataPage;
4042
import org.apache.parquet.column.page.DataPageV1;
43+
import org.apache.parquet.column.page.DictionaryPage;
4144
import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor;
4245
import org.apache.parquet.hadoop.ColumnChunkPageReadStore.ColumnChunkPageReader;
46+
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
4347
import org.apache.parquet.schema.PrimitiveType;
4448
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
4549
import org.apache.parquet.schema.Type.Repetition;
@@ -80,6 +84,63 @@ public void heapDataPagesAreNotRetainedUntilRowGroupClose() throws Exception {
8084
}
8185
}
8286

87+
@Test
88+
public void stableDecompressorOutputIsNotCopied() throws Exception {
89+
BytesInput pageBytes = BytesInput.from(new byte[] {1, 2, 3, 4});
90+
BytesInput dictionaryBytes = BytesInput.from(new byte[] {5, 6, 7, 8});
91+
try (ColumnChunkPageReadStore store = new ColumnChunkPageReadStore(1L)) {
92+
ColumnChunkPageReader reader = newReader(CodecFactory.NO_OP_DECOMPRESSOR, pageBytes, dictionaryBytes);
93+
store.addColumn(COLUMN, reader);
94+
assertThat(((DataPageV1) reader.readPage()).getBytes()).isSameAs(pageBytes);
95+
assertThat(reader.readDictionaryPage().getBytes()).isSameAs(dictionaryBytes);
96+
}
97+
}
98+
99+
@Test
100+
public void otherDecompressorOutputIsCopied() throws Exception {
101+
BytesInput pageBytes = BytesInput.from(new byte[] {1, 2, 3, 4});
102+
BytesInput dictionaryBytes = BytesInput.from(new byte[] {5, 6, 7, 8});
103+
try (ColumnChunkPageReadStore store = new ColumnChunkPageReadStore(1L)) {
104+
ColumnChunkPageReader reader = newReader(NOOP_DECOMPRESSOR, pageBytes, dictionaryBytes);
105+
store.addColumn(COLUMN, reader);
106+
BytesInput page = ((DataPageV1) reader.readPage()).getBytes();
107+
assertThat(page).isNotSameAs(pageBytes);
108+
assertThat(page.toByteArray()).isEqualTo(pageBytes.toByteArray());
109+
BytesInput dictionary = reader.readDictionaryPage().getBytes();
110+
assertThat(dictionary).isNotSameAs(dictionaryBytes);
111+
assertThat(dictionary.toByteArray()).isEqualTo(dictionaryBytes.toByteArray());
112+
}
113+
}
114+
115+
@Test
116+
public void onlyDecompressorsWithOwnedOutputReturnStableBytes() {
117+
Configuration conf = new Configuration();
118+
CodecFactory heap = new CodecFactory(conf, 1024);
119+
CodecFactory direct =
120+
CodecFactory.createDirectCodecFactory(conf, DirectByteBufferAllocator.getInstance(), 1024);
121+
try {
122+
assertThat(heap.getDecompressor(CompressionCodecName.UNCOMPRESSED).returnsStableBytes())
123+
.isTrue();
124+
assertThat(heap.getDecompressor(CompressionCodecName.ZSTD).returnsStableBytes())
125+
.isTrue();
126+
assertThat(heap.getDecompressor(CompressionCodecName.LZ4_RAW).returnsStableBytes())
127+
.isTrue();
128+
// Lazy streams tied to the decompressor
129+
assertThat(heap.getDecompressor(CompressionCodecName.SNAPPY).returnsStableBytes())
130+
.isFalse();
131+
assertThat(heap.getDecompressor(CompressionCodecName.GZIP).returnsStableBytes())
132+
.isFalse();
133+
// Reuses its output buffer (GH-3150)
134+
assertThat(direct.getDecompressor(CompressionCodecName.SNAPPY).returnsStableBytes())
135+
.isFalse();
136+
assertThat(direct.getDecompressor(CompressionCodecName.ZSTD).returnsStableBytes())
137+
.isFalse();
138+
} finally {
139+
heap.release();
140+
direct.release();
141+
}
142+
}
143+
83144
@Test
84145
public void closeWithoutSetReleaserDoesNotThrow() {
85146
try (TrackingByteBufferAllocator allocator = TrackingByteBufferAllocator.wrap(new HeapByteBufferAllocator())) {
@@ -159,6 +220,23 @@ public boolean isDirect() {
159220
};
160221
}
161222

223+
private static ColumnChunkPageReader newReader(
224+
BytesInputDecompressor decompressor, BytesInput pageBytes, BytesInput dictionaryBytes) throws Exception {
225+
DataPageV1 page = new DataPageV1(pageBytes, 1, (int) pageBytes.size(), null, RLE, RLE, PLAIN);
226+
DictionaryPage dictionary = new DictionaryPage(dictionaryBytes, (int) dictionaryBytes.size(), 1, PLAIN);
227+
return new ColumnChunkPageReader(
228+
decompressor,
229+
Collections.<DataPage>singletonList(page),
230+
dictionary,
231+
null,
232+
1L,
233+
null,
234+
null,
235+
0,
236+
0,
237+
ParquetReadOptions.builder().build());
238+
}
239+
162240
private static ColumnChunkPageReader newReaderWithoutPages(ParquetReadOptions options) {
163241
return new ColumnChunkPageReader(
164242
NOOP_DECOMPRESSOR, Collections.<DataPage>emptyList(), null, null, 0L, null, null, 0, 0, options);

0 commit comments

Comments
 (0)