diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/CodecFactory.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/CodecFactory.java index 14b8897b21..765aea0397 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/CodecFactory.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/CodecFactory.java @@ -88,6 +88,11 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) { return bytes; } + @Override + boolean returnsStableBytes() { + return true; // returns the input, which the caller owns + } + @Override public void release() {} }; @@ -173,6 +178,15 @@ class HeapBytesDecompressor extends BytesDecompressor { decompressor = CodecPool.getDecompressor(codec); } + private boolean materializesEagerly() { + return codec instanceof ZstandardCodec || codec instanceof Lz4RawCodec; + } + + @Override + boolean returnsStableBytes() { + return materializesEagerly(); // a fresh array; the lazy stream of other codecs is tied to the decompressor + } + @Override public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOException { final BytesInput decompressed; @@ -192,7 +206,7 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOEx // LZ4_RAW: requires one-shot decompression; the lazy StreamBytesInput.writeInto() path reads via // Channels.newChannel() in ~8KB chunks, causing the decompressor to be called with an undersized // output buffer (see #3478). - if (codec instanceof ZstandardCodec || codec instanceof Lz4RawCodec) { + if (materializesEagerly()) { decompressed = BytesInput.copy(BytesInput.from(is, decompressedSize)); is.close(); } else { @@ -514,5 +528,14 @@ public abstract void decompress(ByteBuffer input, int compressedSize, ByteBuffer throws IOException; public abstract void release(); + + /** + * Whether the {@link BytesInput} returned by {@link #decompress(BytesInput, int)} stays valid when this + * decompressor is used again, so callers need not copy it. Package-private: only implementations in this + * package can opt in; any other implementation is assumed to reuse its buffers. + */ + boolean returnsStableBytes() { + return false; + } } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index a21fec9ab8..d15f9c4eae 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -177,8 +177,7 @@ public DataPage visit(DataPageV1 dataPageV1) { bytes = BytesInput.from(blockDecryptor.decrypt(bytes.toByteArray(), dataPageAAD)); } long start = System.nanoTime(); - decompressed = - BytesInput.copy(decompressor.decompress(bytes, dataPageV1.getUncompressedSize())); + decompressed = ownedCopy(decompressor.decompress(bytes, dataPageV1.getUncompressedSize())); setDecompressMetrics(bytes, start); } @@ -258,7 +257,7 @@ public DataPage visit(DataPageV2 dataPageV2) { - dataPageV2.getDefinitionLevels().size() - dataPageV2.getRepetitionLevels().size()); long start = System.nanoTime(); - pageBytes = BytesInput.copy(decompressor.decompress(pageBytes, uncompressedSize)); + pageBytes = ownedCopy(decompressor.decompress(pageBytes, uncompressedSize)); setDecompressMetrics(pageBytes, start); } } @@ -297,6 +296,24 @@ public DataPage visit(DataPageV2 dataPageV2) { }); } + /** + * The {@link BytesInputDecompressor} contract allows the returned bytes to be overwritten by the next call, but + * pages are kept after the decompressor moves on (GH-3150), so they are copied unless the decompressor guarantees + * stable output. + */ + private BytesInput ownedCopy(BytesInput decompressed) throws IOException { + return returnsStableBytes() ? decompressed : BytesInput.copy(decompressed); + } + + private BytesInput ownedCopy(BytesInput decompressed, ByteBufferReleaser releaser) throws IOException { + return returnsStableBytes() ? decompressed : decompressed.copy(releaser); + } + + private boolean returnsStableBytes() { + return decompressor instanceof CodecFactory.BytesDecompressor + && ((CodecFactory.BytesDecompressor) decompressor).returnsStableBytes(); + } + private void setDecompressMetrics(BytesInput bytes, long start) { final ParquetMetricsCallback metricsCallback = options.getMetricsCallback(); if (metricsCallback != null) { @@ -327,9 +344,9 @@ public DictionaryPage readDictionaryPage() { long start = System.nanoTime(); setDecompressMetrics(bytes, start); DictionaryPage decompressedPage = new DictionaryPage( - decompressor - .decompress(bytes, compressedDictionaryPage.getUncompressedSize()) - .copy(releaser), + ownedCopy( + decompressor.decompress(bytes, compressedDictionaryPage.getUncompressedSize()), + releaser), compressedDictionaryPage.getDictionarySize(), compressedDictionaryPage.getEncoding()); if (compressedDictionaryPage.getCrc().isPresent()) { diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageReadStore.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageReadStore.java index d83bae46bd..775bab8c10 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageReadStore.java @@ -29,17 +29,21 @@ import java.nio.ByteBuffer; import java.util.Collections; +import org.apache.hadoop.conf.Configuration; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.bytes.ByteBufferReleaser; import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.DirectByteBufferAllocator; import org.apache.parquet.bytes.HeapByteBufferAllocator; import org.apache.parquet.bytes.TrackingByteBufferAllocator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor; import org.apache.parquet.hadoop.ColumnChunkPageReadStore.ColumnChunkPageReader; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Type.Repetition; @@ -80,6 +84,63 @@ public void heapDataPagesAreNotRetainedUntilRowGroupClose() throws Exception { } } + @Test + public void stableDecompressorOutputIsNotCopied() throws Exception { + BytesInput pageBytes = BytesInput.from(new byte[] {1, 2, 3, 4}); + BytesInput dictionaryBytes = BytesInput.from(new byte[] {5, 6, 7, 8}); + try (ColumnChunkPageReadStore store = new ColumnChunkPageReadStore(1L)) { + ColumnChunkPageReader reader = newReader(CodecFactory.NO_OP_DECOMPRESSOR, pageBytes, dictionaryBytes); + store.addColumn(COLUMN, reader); + assertThat(((DataPageV1) reader.readPage()).getBytes()).isSameAs(pageBytes); + assertThat(reader.readDictionaryPage().getBytes()).isSameAs(dictionaryBytes); + } + } + + @Test + public void otherDecompressorOutputIsCopied() throws Exception { + BytesInput pageBytes = BytesInput.from(new byte[] {1, 2, 3, 4}); + BytesInput dictionaryBytes = BytesInput.from(new byte[] {5, 6, 7, 8}); + try (ColumnChunkPageReadStore store = new ColumnChunkPageReadStore(1L)) { + ColumnChunkPageReader reader = newReader(NOOP_DECOMPRESSOR, pageBytes, dictionaryBytes); + store.addColumn(COLUMN, reader); + BytesInput page = ((DataPageV1) reader.readPage()).getBytes(); + assertThat(page).isNotSameAs(pageBytes); + assertThat(page.toByteArray()).isEqualTo(pageBytes.toByteArray()); + BytesInput dictionary = reader.readDictionaryPage().getBytes(); + assertThat(dictionary).isNotSameAs(dictionaryBytes); + assertThat(dictionary.toByteArray()).isEqualTo(dictionaryBytes.toByteArray()); + } + } + + @Test + public void onlyDecompressorsWithOwnedOutputReturnStableBytes() { + Configuration conf = new Configuration(); + CodecFactory heap = new CodecFactory(conf, 1024); + CodecFactory direct = + CodecFactory.createDirectCodecFactory(conf, DirectByteBufferAllocator.getInstance(), 1024); + try { + assertThat(heap.getDecompressor(CompressionCodecName.UNCOMPRESSED).returnsStableBytes()) + .isTrue(); + assertThat(heap.getDecompressor(CompressionCodecName.ZSTD).returnsStableBytes()) + .isTrue(); + assertThat(heap.getDecompressor(CompressionCodecName.LZ4_RAW).returnsStableBytes()) + .isTrue(); + // Lazy streams tied to the decompressor + assertThat(heap.getDecompressor(CompressionCodecName.SNAPPY).returnsStableBytes()) + .isFalse(); + assertThat(heap.getDecompressor(CompressionCodecName.GZIP).returnsStableBytes()) + .isFalse(); + // Reuses its output buffer (GH-3150) + assertThat(direct.getDecompressor(CompressionCodecName.SNAPPY).returnsStableBytes()) + .isFalse(); + assertThat(direct.getDecompressor(CompressionCodecName.ZSTD).returnsStableBytes()) + .isFalse(); + } finally { + heap.release(); + direct.release(); + } + } + @Test public void closeWithoutSetReleaserDoesNotThrow() { try (TrackingByteBufferAllocator allocator = TrackingByteBufferAllocator.wrap(new HeapByteBufferAllocator())) { @@ -159,6 +220,23 @@ public boolean isDirect() { }; } + private static ColumnChunkPageReader newReader( + BytesInputDecompressor decompressor, BytesInput pageBytes, BytesInput dictionaryBytes) throws Exception { + DataPageV1 page = new DataPageV1(pageBytes, 1, (int) pageBytes.size(), null, RLE, RLE, PLAIN); + DictionaryPage dictionary = new DictionaryPage(dictionaryBytes, (int) dictionaryBytes.size(), 1, PLAIN); + return new ColumnChunkPageReader( + decompressor, + Collections.singletonList(page), + dictionary, + null, + 1L, + null, + null, + 0, + 0, + ParquetReadOptions.builder().build()); + } + private static ColumnChunkPageReader newReaderWithoutPages(ParquetReadOptions options) { return new ColumnChunkPageReader( NOOP_DECOMPRESSOR, Collections.emptyList(), null, null, 0L, null, null, 0, 0, options);