Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) {
return bytes;
}

@Override
boolean returnsStableBytes() {
return true; // returns the input, which the caller owns
}

@Override
public void release() {}
};
Expand Down Expand Up @@ -173,6 +178,15 @@ class HeapBytesDecompressor extends BytesDecompressor {
decompressor = CodecPool.getDecompressor(codec);
}

private boolean materializesEagerly() {
return codec instanceof ZstandardCodec || codec instanceof Lz4RawCodec;
}

@Override
boolean returnsStableBytes() {
return materializesEagerly(); // a fresh array; the lazy stream of other codecs is tied to the decompressor
}

@Override
public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOException {
final BytesInput decompressed;
Expand All @@ -192,7 +206,7 @@ public BytesInput decompress(BytesInput bytes, int decompressedSize) throws IOEx
// LZ4_RAW: requires one-shot decompression; the lazy StreamBytesInput.writeInto() path reads via
// Channels.newChannel() in ~8KB chunks, causing the decompressor to be called with an undersized
// output buffer (see #3478).
if (codec instanceof ZstandardCodec || codec instanceof Lz4RawCodec) {
if (materializesEagerly()) {
decompressed = BytesInput.copy(BytesInput.from(is, decompressedSize));
is.close();
} else {
Expand Down Expand Up @@ -514,5 +528,14 @@ public abstract void decompress(ByteBuffer input, int compressedSize, ByteBuffer
throws IOException;

public abstract void release();

/**
* Whether the {@link BytesInput} returned by {@link #decompress(BytesInput, int)} stays valid when this
* decompressor is used again, so callers need not copy it. Package-private: only implementations in this
* package can opt in; any other implementation is assumed to reuse its buffers.
*/
boolean returnsStableBytes() {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,7 @@ public DataPage visit(DataPageV1 dataPageV1) {
bytes = BytesInput.from(blockDecryptor.decrypt(bytes.toByteArray(), dataPageAAD));
}
long start = System.nanoTime();
decompressed =
BytesInput.copy(decompressor.decompress(bytes, dataPageV1.getUncompressedSize()));
decompressed = ownedCopy(decompressor.decompress(bytes, dataPageV1.getUncompressedSize()));
setDecompressMetrics(bytes, start);
}

Expand Down Expand Up @@ -258,7 +257,7 @@ public DataPage visit(DataPageV2 dataPageV2) {
- dataPageV2.getDefinitionLevels().size()
- dataPageV2.getRepetitionLevels().size());
long start = System.nanoTime();
pageBytes = BytesInput.copy(decompressor.decompress(pageBytes, uncompressedSize));
pageBytes = ownedCopy(decompressor.decompress(pageBytes, uncompressedSize));
setDecompressMetrics(pageBytes, start);
}
}
Expand Down Expand Up @@ -297,6 +296,24 @@ public DataPage visit(DataPageV2 dataPageV2) {
});
}

/**
* The {@link BytesInputDecompressor} contract allows the returned bytes to be overwritten by the next call, but
* pages are kept after the decompressor moves on (GH-3150), so they are copied unless the decompressor guarantees
* stable output.
*/
private BytesInput ownedCopy(BytesInput decompressed) throws IOException {
return returnsStableBytes() ? decompressed : BytesInput.copy(decompressed);
}

private BytesInput ownedCopy(BytesInput decompressed, ByteBufferReleaser releaser) throws IOException {
return returnsStableBytes() ? decompressed : decompressed.copy(releaser);
}

private boolean returnsStableBytes() {
return decompressor instanceof CodecFactory.BytesDecompressor
&& ((CodecFactory.BytesDecompressor) decompressor).returnsStableBytes();
}

private void setDecompressMetrics(BytesInput bytes, long start) {
final ParquetMetricsCallback metricsCallback = options.getMetricsCallback();
if (metricsCallback != null) {
Expand Down Expand Up @@ -327,9 +344,9 @@ public DictionaryPage readDictionaryPage() {
long start = System.nanoTime();
setDecompressMetrics(bytes, start);
DictionaryPage decompressedPage = new DictionaryPage(
decompressor
.decompress(bytes, compressedDictionaryPage.getUncompressedSize())
.copy(releaser),
ownedCopy(
decompressor.decompress(bytes, compressedDictionaryPage.getUncompressedSize()),
releaser),
compressedDictionaryPage.getDictionarySize(),
compressedDictionaryPage.getEncoding());
if (compressedDictionaryPage.getCrc().isPresent()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,21 @@

import java.nio.ByteBuffer;
import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.ParquetReadOptions;
import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.bytes.ByteBufferReleaser;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.bytes.DirectByteBufferAllocator;
import org.apache.parquet.bytes.HeapByteBufferAllocator;
import org.apache.parquet.bytes.TrackingByteBufferAllocator;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.page.DataPage;
import org.apache.parquet.column.page.DataPageV1;
import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor;
import org.apache.parquet.hadoop.ColumnChunkPageReadStore.ColumnChunkPageReader;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type.Repetition;
Expand Down Expand Up @@ -80,6 +84,63 @@ public void heapDataPagesAreNotRetainedUntilRowGroupClose() throws Exception {
}
}

@Test
public void stableDecompressorOutputIsNotCopied() throws Exception {
BytesInput pageBytes = BytesInput.from(new byte[] {1, 2, 3, 4});
BytesInput dictionaryBytes = BytesInput.from(new byte[] {5, 6, 7, 8});
try (ColumnChunkPageReadStore store = new ColumnChunkPageReadStore(1L)) {
ColumnChunkPageReader reader = newReader(CodecFactory.NO_OP_DECOMPRESSOR, pageBytes, dictionaryBytes);
store.addColumn(COLUMN, reader);
assertThat(((DataPageV1) reader.readPage()).getBytes()).isSameAs(pageBytes);
assertThat(reader.readDictionaryPage().getBytes()).isSameAs(dictionaryBytes);
}
}

@Test
public void otherDecompressorOutputIsCopied() throws Exception {
BytesInput pageBytes = BytesInput.from(new byte[] {1, 2, 3, 4});
BytesInput dictionaryBytes = BytesInput.from(new byte[] {5, 6, 7, 8});
try (ColumnChunkPageReadStore store = new ColumnChunkPageReadStore(1L)) {
ColumnChunkPageReader reader = newReader(NOOP_DECOMPRESSOR, pageBytes, dictionaryBytes);
store.addColumn(COLUMN, reader);
BytesInput page = ((DataPageV1) reader.readPage()).getBytes();
assertThat(page).isNotSameAs(pageBytes);
assertThat(page.toByteArray()).isEqualTo(pageBytes.toByteArray());
BytesInput dictionary = reader.readDictionaryPage().getBytes();
assertThat(dictionary).isNotSameAs(dictionaryBytes);
assertThat(dictionary.toByteArray()).isEqualTo(dictionaryBytes.toByteArray());
}
}

@Test
public void onlyDecompressorsWithOwnedOutputReturnStableBytes() {
Configuration conf = new Configuration();
CodecFactory heap = new CodecFactory(conf, 1024);
CodecFactory direct =
CodecFactory.createDirectCodecFactory(conf, DirectByteBufferAllocator.getInstance(), 1024);
try {
assertThat(heap.getDecompressor(CompressionCodecName.UNCOMPRESSED).returnsStableBytes())
.isTrue();
assertThat(heap.getDecompressor(CompressionCodecName.ZSTD).returnsStableBytes())
.isTrue();
assertThat(heap.getDecompressor(CompressionCodecName.LZ4_RAW).returnsStableBytes())
.isTrue();
// Lazy streams tied to the decompressor
assertThat(heap.getDecompressor(CompressionCodecName.SNAPPY).returnsStableBytes())
.isFalse();
assertThat(heap.getDecompressor(CompressionCodecName.GZIP).returnsStableBytes())
.isFalse();
// Reuses its output buffer (GH-3150)
assertThat(direct.getDecompressor(CompressionCodecName.SNAPPY).returnsStableBytes())
.isFalse();
assertThat(direct.getDecompressor(CompressionCodecName.ZSTD).returnsStableBytes())
.isFalse();
} finally {
heap.release();
direct.release();
}
}

@Test
public void closeWithoutSetReleaserDoesNotThrow() {
try (TrackingByteBufferAllocator allocator = TrackingByteBufferAllocator.wrap(new HeapByteBufferAllocator())) {
Expand Down Expand Up @@ -159,6 +220,23 @@ public boolean isDirect() {
};
}

private static ColumnChunkPageReader newReader(
BytesInputDecompressor decompressor, BytesInput pageBytes, BytesInput dictionaryBytes) throws Exception {
DataPageV1 page = new DataPageV1(pageBytes, 1, (int) pageBytes.size(), null, RLE, RLE, PLAIN);
DictionaryPage dictionary = new DictionaryPage(dictionaryBytes, (int) dictionaryBytes.size(), 1, PLAIN);
return new ColumnChunkPageReader(
decompressor,
Collections.<DataPage>singletonList(page),
dictionary,
null,
1L,
null,
null,
0,
0,
ParquetReadOptions.builder().build());
}

private static ColumnChunkPageReader newReaderWithoutPages(ParquetReadOptions options) {
return new ColumnChunkPageReader(
NOOP_DECOMPRESSOR, Collections.<DataPage>emptyList(), null, null, 0L, null, null, 0, 0, options);
Expand Down
Loading