Skip to content

Commit eb6bfcf

Browse files
authored
GH-3869: Preserve destination state in single-buffer reads (#3870)
1 parent 567c0ec commit eb6bfcf

3 files changed

Lines changed: 223 additions & 1 deletion

File tree

‎parquet-common/src/main/java/org/apache/parquet/bytes/ByteBufferInputStream.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,14 @@ public void skipFully(long n) throws IOException {
100100
}
101101
}
102102

103+
/**
104+
* Copies bytes from this stream into {@code out}, reading at most {@code out.remaining()} bytes. The position of
105+
* {@code out} is advanced by the number of bytes copied and its limit is unchanged.
106+
*
107+
* @param out the buffer to copy bytes into
108+
* @return the number of bytes copied, {@code 0} if {@code out} has no remaining space, or {@code -1} if this stream
109+
* is exhausted
110+
*/
103111
public int read(ByteBuffer out) {
104112
return delegate.read(out);
105113
}

‎parquet-common/src/main/java/org/apache/parquet/bytes/SingleBufferInputStream.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,14 @@ public long skip(long n) {
9090

9191
@Override
9292
public int read(ByteBuffer out) {
93+
if (!out.hasRemaining()) {
94+
return 0;
95+
}
96+
97+
if (buffer.remaining() <= 0) {
98+
return -1;
99+
}
100+
93101
int bytesToCopy;
94102
ByteBuffer copyBuffer;
95103
if (buffer.remaining() <= out.remaining()) {
@@ -105,7 +113,6 @@ public int read(ByteBuffer out) {
105113
}
106114

107115
out.put(copyBuffer);
108-
out.flip();
109116

110117
return bytesToCopy;
111118
}
Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.parquet.bytes;
21+
22+
import static org.assertj.core.api.Assertions.assertThat;
23+
24+
import java.nio.ByteBuffer;
25+
import java.util.Arrays;
26+
import java.util.List;
27+
import java.util.stream.Stream;
28+
import org.junit.jupiter.params.ParameterizedTest;
29+
import org.junit.jupiter.params.provider.Arguments;
30+
import org.junit.jupiter.params.provider.MethodSource;
31+
32+
/**
33+
* Tests {@link ByteBufferInputStream#read(ByteBuffer)} appending into the caller's destination buffer for both single
34+
* buffer and multi buffer streams created through the public wrap entry points.
35+
*/
36+
public class TestByteBufferInputStreamRead {
37+
38+
private static final byte PREFIX_BYTE = (byte) 0x11;
39+
private static final byte SUFFIX_BYTE = (byte) 0x77;
40+
private static final int PREFIX_LENGTH = 3;
41+
private static final int DEST_CAPACITY = 16;
42+
private static final int DEST_LIMIT = 8;
43+
private static final int SLICE_MARGIN = 4;
44+
45+
private enum OutputKind {
46+
HEAP,
47+
DIRECT,
48+
SLICED
49+
}
50+
51+
private enum SourceKind {
52+
SINGLE,
53+
MULTI
54+
}
55+
56+
static Stream<Arguments> kinds() {
57+
return Stream.of(OutputKind.values())
58+
.flatMap(output -> Stream.of(SourceKind.values()).map(source -> Arguments.of(output, source)));
59+
}
60+
61+
@ParameterizedTest(name = "{0} {1}")
62+
@MethodSource("kinds")
63+
public void partialCopyAdvancesDestinationPosition(OutputKind output, SourceKind source) {
64+
byte[] data = data(10, 1);
65+
int copied = DEST_LIMIT - PREFIX_LENGTH;
66+
ByteBuffer dest = destination(output, DEST_LIMIT, PREFIX_LENGTH);
67+
68+
assertRead(output + " " + source, stream(source, data, false), dest, copied);
69+
70+
assertContent(dest, output, source, Arrays.copyOf(data, copied));
71+
}
72+
73+
@ParameterizedTest(name = "{0} {1}")
74+
@MethodSource("kinds")
75+
public void fullSourceCopyKeepsLimitAndRoomForAppendedRead(OutputKind output, SourceKind source) {
76+
byte[] first = data(4, 1);
77+
byte[] second = data(3, 11);
78+
ByteBuffer dest = destination(output, DEST_CAPACITY, PREFIX_LENGTH);
79+
80+
assertRead(output + " " + source + " first read", stream(source, first, true), dest, first.length);
81+
assertRead(output + " " + source + " second read", stream(source, second, true), dest, second.length);
82+
83+
assertContent(dest, output, source, first, second);
84+
}
85+
86+
@ParameterizedTest(name = "{0} {1}")
87+
@MethodSource("kinds")
88+
public void readIntoExhaustedDestinationReturnsZeroWithoutMutatingIt(OutputKind output, SourceKind source) {
89+
byte[] data = data(10, 1);
90+
ByteBuffer dest = destination(output, DEST_LIMIT, DEST_LIMIT);
91+
ByteBufferInputStream in = stream(source, data, false);
92+
93+
assertThat(in.read(dest)).as("%s %s: bytes copied", output, source).isZero();
94+
assertThat(dest.position())
95+
.as("%s %s: destination position", output, source)
96+
.isEqualTo(DEST_LIMIT);
97+
assertThat(dest.limit()).as("%s %s: destination limit", output, source).isEqualTo(DEST_LIMIT);
98+
assertThat(in.position()).as("%s %s: source position", output, source).isZero();
99+
assertThat(in.available()).as("%s %s: source remaining", output, source).isEqualTo(data.length);
100+
}
101+
102+
@ParameterizedTest(name = "{0} {1}")
103+
@MethodSource("kinds")
104+
public void readAtEndOfStreamReturnsMinusOneWithoutMutatingDestination(OutputKind output, SourceKind source)
105+
throws Exception {
106+
byte[] data = data(10, 1);
107+
ByteBufferInputStream in = stream(source, data, true);
108+
ByteBuffer dest = destination(output, DEST_LIMIT, PREFIX_LENGTH);
109+
110+
assertThat(in.read(new byte[data.length]))
111+
.as("%s %s: drained bytes", output, source)
112+
.isEqualTo(data.length);
113+
assertThat(in.read(dest)).as("%s %s: bytes copied", output, source).isEqualTo(-1);
114+
assertThat(dest.position())
115+
.as("%s %s: destination position", output, source)
116+
.isEqualTo(PREFIX_LENGTH);
117+
assertThat(dest.limit()).as("%s %s: destination limit", output, source).isEqualTo(DEST_LIMIT);
118+
assertThat(in.position()).as("%s %s: source position", output, source).isEqualTo(data.length);
119+
assertThat(in.available()).as("%s %s: source remaining", output, source).isZero();
120+
}
121+
122+
private static void assertRead(String description, ByteBufferInputStream in, ByteBuffer dest, int expectedCopied) {
123+
int destStart = dest.position();
124+
int destLimit = dest.limit();
125+
long sourceStart = in.position();
126+
int sourceRemaining = in.available();
127+
128+
assertThat(in.read(dest)).as("%s: bytes copied", description).isEqualTo(expectedCopied);
129+
assertThat(dest.position()).as("%s: destination position", description).isEqualTo(destStart + expectedCopied);
130+
assertThat(dest.limit()).as("%s: destination limit", description).isEqualTo(destLimit);
131+
assertThat(in.position()).as("%s: source position", description).isEqualTo(sourceStart + expectedCopied);
132+
assertThat(in.available()).as("%s: source remaining", description).isEqualTo(sourceRemaining - expectedCopied);
133+
}
134+
135+
private static void assertContent(ByteBuffer dest, OutputKind output, SourceKind source, byte[]... payloads) {
136+
byte[] expected = new byte
137+
[PREFIX_LENGTH
138+
+ Arrays.stream(payloads)
139+
.mapToInt(payload -> payload.length)
140+
.sum()];
141+
Arrays.fill(expected, 0, PREFIX_LENGTH, PREFIX_BYTE);
142+
int offset = PREFIX_LENGTH;
143+
for (byte[] payload : payloads) {
144+
System.arraycopy(payload, 0, expected, offset, payload.length);
145+
offset += payload.length;
146+
}
147+
148+
dest.flip();
149+
byte[] actual = new byte[dest.remaining()];
150+
dest.get(actual);
151+
assertThat(actual)
152+
.as("%s %s: prefilled prefix followed by the copied source bytes", output, source)
153+
.containsExactly(expected);
154+
155+
ByteBuffer fullDestination = dest.duplicate();
156+
fullDestination.clear();
157+
for (int i = dest.limit(); i < dest.capacity(); i++) {
158+
assertThat(fullDestination.get(i))
159+
.as("%s %s: untouched destination byte %d", output, source, i)
160+
.isEqualTo(SUFFIX_BYTE);
161+
}
162+
}
163+
164+
private static ByteBuffer destination(OutputKind output, int limit, int prefixLength) {
165+
ByteBuffer dest = ByteBuffer.allocate(DEST_CAPACITY);
166+
if (output == OutputKind.DIRECT) {
167+
dest = ByteBuffer.allocateDirect(DEST_CAPACITY);
168+
} else if (output == OutputKind.SLICED) {
169+
ByteBuffer parent = ByteBuffer.allocate(DEST_CAPACITY + 2 * SLICE_MARGIN);
170+
parent.position(SLICE_MARGIN);
171+
parent.limit(SLICE_MARGIN + DEST_CAPACITY);
172+
dest = parent.slice();
173+
}
174+
175+
for (int i = 0; i < dest.capacity(); i++) {
176+
dest.put(i, SUFFIX_BYTE);
177+
}
178+
for (int i = 0; i < prefixLength; i++) {
179+
dest.put(i, PREFIX_BYTE);
180+
}
181+
dest.limit(limit);
182+
dest.position(prefixLength);
183+
184+
return dest;
185+
}
186+
187+
private static ByteBufferInputStream stream(SourceKind source, byte[] data, boolean viaList) {
188+
if (source == SourceKind.SINGLE) {
189+
ByteBuffer whole = ByteBuffer.wrap(data);
190+
return viaList ? ByteBufferInputStream.wrap(List.of(whole)) : ByteBufferInputStream.wrap(whole);
191+
}
192+
193+
// the first buffer holds more than half of the data so that a small destination forces a partial copy
194+
int split = data.length / 2 + 1;
195+
ByteBuffer first = ByteBuffer.wrap(data, 0, split);
196+
ByteBuffer second = ByteBuffer.wrap(data, split, data.length - split);
197+
return viaList ? ByteBufferInputStream.wrap(List.of(first, second)) : ByteBufferInputStream.wrap(first, second);
198+
}
199+
200+
private static byte[] data(int length, int firstValue) {
201+
byte[] data = new byte[length];
202+
for (int i = 0; i < length; i++) {
203+
data[i] = (byte) (firstValue + i);
204+
}
205+
return data;
206+
}
207+
}

0 commit comments

Comments
 (0)