From 66ea5aea7e9e805092b6eb15410b5e6d7e080431 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Wed, 19 Aug 2026 09:24:14 +0800 Subject: [PATCH] [arrow] Fix nanosecond timestamp conversion for pre-epoch values convertEpochToTimestamp splits a nanosecond epoch with / and %, which truncate toward zero, so a pre-epoch value yields a negative nano-of-millisecond and Timestamp's constructor rejects it. Timestamp#fromMicros and the Parquet reader both use Math.floorDiv. Written with Claude Code; reasoning and verification are mine. --- .../Arrow2PaimonVectorConverter.java | 3 +- .../arrow/vector/ArrowFormatWriterTest.java | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java index e3a9a53d51bc..4ce6871809f1 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java @@ -462,7 +462,8 @@ private Timestamp convertEpochToTimestamp(long value, int precision) { } else if (precision >= 4 && precision <= 6) { return Timestamp.fromMicros(value); } else { - return Timestamp.fromEpochMillis(value / 1_000_000, (int) (value % 1_000_000)); + return Timestamp.fromEpochMillis( + Math.floorDiv(value, 1_000_000L), (int) Math.floorMod(value, 1_000_000L)); } } diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java index fd257450aaa9..d8cfd8fde944 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java @@ -53,6 +53,8 @@ import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeNanoVector; import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -587,6 +589,43 @@ public void testArrowBundleRecordsWithTimeAndFixedBinaryVectors() { } } + @Test + public void testArrowBundleRecordsWithPreEpochNanoTimestamps() { + RowType rowType = + RowType.of( + new DataField(0, "ts_nano", DataTypes.TIMESTAMP(9)), + new DataField( + 1, "ts_ltz_nano", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9))); + + // 1969-12-31T23:59:59.999999999, i.e. one nanosecond before the epoch + long nanos = -1L; + + try (RootAllocator allocator = new RootAllocator()) { + TimeStampNanoVector tsVector = new TimeStampNanoVector("ts_nano", allocator); + tsVector.allocateNew(1); + tsVector.setSafe(0, nanos); + tsVector.setValueCount(1); + + TimeStampNanoTZVector tsLtzVector = + new TimeStampNanoTZVector("ts_ltz_nano", allocator, "UTC"); + tsLtzVector.allocateNew(1); + tsLtzVector.setSafe(0, nanos); + tsLtzVector.setValueCount(1); + + List vectors = Arrays.asList(tsVector, tsLtzVector); + try (VectorSchemaRoot vectorSchemaRoot = new VectorSchemaRoot(vectors)) { + vectorSchemaRoot.setRowCount(1); + + Iterator iterator = + new ArrowBundleRecords(vectorSchemaRoot, rowType, true).iterator(); + InternalRow row = iterator.next(); + Timestamp expected = Timestamp.fromEpochMillis(-1, 999_999); + assertThat(row.getTimestamp(0, 9)).isEqualTo(expected); + assertThat(row.getTimestamp(1, 9)).isEqualTo(expected); + } + } + } + @Test public void testCWriter() { try (ArrowFormatCWriter writer = new ArrowFormatCWriter(PRIMITIVE_TYPE, 4096, true)) {