From da09d6f1ca35028f23a48390852dc1034d6487a8 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Tue, 10 Mar 2026 18:24:53 +0530 Subject: [PATCH 1/6] TEZ-4677: Upgrades Apache Tez to compile and run with JDK 25 --- build-tools/docker/Dockerfile | 2 +- pom.xml | 5 +++-- tez-dist/src/docker/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 12fa58d890..63bee8dba2 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -106,7 +106,7 @@ RUN apk add --no-cache curl tar \ #################### # Switching to eclipse-temurin to resolves cross-platform amd64/arm64 architecture issues -FROM eclipse-temurin:21-jdk-noble AS tezbase +FROM eclipse-temurin:25-jdk-noble AS tezbase WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/pom.xml b/pom.xml index ccdc48d2d1..332084a724 100644 --- a/pom.xml +++ b/pom.xml @@ -47,8 +47,9 @@ - 21 - 21 + 25 + 25 + 3.6.2 true false UTF-8 diff --git a/tez-dist/src/docker/Dockerfile b/tez-dist/src/docker/Dockerfile index 3d13f5e2b9..1db272e273 100644 --- a/tez-dist/src/docker/Dockerfile +++ b/tez-dist/src/docker/Dockerfile @@ -34,7 +34,7 @@ RUN mkdir -p /opt/tez \ -C /opt/tez \ && rm -rf /opt/tez-$TEZ_VERSION.tar.gz -FROM eclipse-temurin:21-jdk-ubi9-minimal AS run +FROM eclipse-temurin:25-jdk-ubi10-minimal AS run ARG UID=1000 ARG TEZ_VERSION From 58adad48cada7523f9f1ca9076e13589346398ab Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Mon, 29 Jun 2026 11:49:25 +0530 Subject: [PATCH 2/6] Fixed javac issues --- .../dag/app/rm/TestLocalTaskScheduler.java | 4 +- .../app/rm/TestLocalTaskSchedulerService.java | 8 +-- .../objectregistry/TestObjectRegistry.java | 6 +-- .../library/utils/FastByteComparisons.java | 52 +++++-------------- .../TestShuffleVertexManager.java | 4 +- .../TestShuffleVertexManagerBase.java | 4 +- .../java/org/apache/tez/test/TestInput.java | 4 +- .../org/apache/tez/test/TestProcessor.java | 4 +- 8 files changed, 29 insertions(+), 57 deletions(-) diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java index 11dd02420a..d55c4bacc0 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java @@ -69,7 +69,7 @@ public void maxTasksAllocationsCannotBeExceeded() { // Allocate up to max tasks for (int i = 0; i < MAX_TASKS; i++) { Priority priority = Priority.newInstance(20); - requestHandler.addAllocateTaskRequest(new Long(i), null, priority, null); + requestHandler.addAllocateTaskRequest((long) i, null, priority, null); requestHandler.dispatchRequest(); requestHandler.allocateTask(); } @@ -80,7 +80,7 @@ public void maxTasksAllocationsCannotBeExceeded() { // Deallocate down to zero for (int i = 0; i < MAX_TASKS; i++) { - requestHandler.addDeallocateTaskRequest(new Long(i)); + requestHandler.addDeallocateTaskRequest((long) i); requestHandler.dispatchRequest(); } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java index fc889bdf77..15bf0d20db 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java @@ -153,10 +153,10 @@ public void preemptDescendantsOnly() { ApplicationId appId = ApplicationId.newInstance(2000, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); - Long parentTask1 = new Long(1); - Long parentTask2 = new Long(2); - Long childTask1 = new Long(3); - Long grandchildTask1 = new Long(4); + Long parentTask1 = 1L; + Long parentTask2 = 2L; + Long childTask1 = 3L; + Long grandchildTask1 = 4L; TaskSchedulerContext mockContext = TestTaskSchedulerHelpers.setupMockTaskSchedulerContext("", 0, "", true, diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java index 472f3d2394..cdcb20b81b 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java @@ -30,9 +30,9 @@ private void testCRUD(ObjectRegistry objectRegistry) { Assert.assertNull(objectRegistry.get("foo")); Assert.assertFalse(objectRegistry.delete("foo")); - Integer one = new Integer(1); - Integer two_1 = new Integer(2); - Integer two_2 = new Integer(3); + Integer one = 1; + Integer two_1 = 2; + Integer two_2 = 3; Assert.assertNull(objectRegistry.cacheForDAG("one", one)); Assert.assertEquals(one, objectRegistry.get("one")); Assert.assertNull(objectRegistry.cacheForDAG("two", two_1)); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java index a392416a04..4f66ec8869 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java @@ -36,16 +36,13 @@ * limitations under the License. */ -import java.lang.reflect.Field; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.nio.ByteOrder; -import java.security.AccessController; -import java.security.PrivilegedAction; import com.google.common.primitives.Longs; import com.google.common.primitives.UnsignedBytes; -import sun.misc.Unsafe; - /** * Same as {@link org.apache.hadoop.io.FastByteComparisons} * @@ -79,10 +76,10 @@ private static Comparer lexicographicalComparerJavaImpl() { /** * Provides a lexicographical comparer implementation; either a Java - * implementation or a faster implementation based on {@link sun.misc.Unsafe}. + * implementation or a faster implementation based on {@link java.lang.invoke.VarHandle}. * *

Uses reflection to gracefully fall back to the Java implementation if - * {@code Unsafe} isn't available. + * {@code VarHandle} isn't available. */ private static class LexicographicalComparerHolder { static final String UNSAFE_COMPARER_NAME = @@ -137,35 +134,12 @@ public int compareTo(byte[] buffer1, int offset1, int length1, private enum UnsafeComparer implements Comparer { INSTANCE; - static final Unsafe theUnsafe; - - /** The offset to the first element in a byte array. */ - static final int BYTE_ARRAY_BASE_OFFSET; - - static { - theUnsafe = (Unsafe) AccessController.doPrivileged( - new PrivilegedAction() { - @Override - public Object run() { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return f.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - // It doesn't matter what we throw; - // it's swallowed in getBestComparer(). - throw new Error(); - } - } - }); - - BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); - - // sanity check - this should never fail - if (theUnsafe.arrayIndexScale(byte[].class) != 1) { - throw new AssertionError(); - } - } + /** + * A VarHandle for reading longs from a byte array using native byte order. + * This replaces the deprecated sun.misc.Unsafe getLong(Object, long) API. + */ + static final VarHandle LONG_VIEW = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); static final boolean littleEndian = ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); @@ -200,8 +174,6 @@ public int compareTo(byte[] buffer1, int offset1, int length1, } int minLength = Math.min(length1, length2); int minWords = minLength / Longs.BYTES; - int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET; - int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET; /* * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a @@ -209,8 +181,8 @@ public int compareTo(byte[] buffer1, int offset1, int length1, * On the other hand, it is substantially faster on 64-bit. */ for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) { - long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i); - long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i); + long lw = (long) LONG_VIEW.get(buffer1, offset1 + i); + long rw = (long) LONG_VIEW.get(buffer2, offset2 + i); long diff = lw ^ rw; if (diff != 0) { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java index 63ab5daed4..7a9680fc12 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java @@ -148,8 +148,8 @@ public void testLargeDataSize() throws IOException { Assert.assertEquals(2, newEdgeManagers.size()); Assert.assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled Assert.assertEquals(2, scheduledTasks.size()); - Assert.assertTrue(scheduledTasks.contains(new Integer(0))); - Assert.assertTrue(scheduledTasks.contains(new Integer(1))); + Assert.assertTrue(scheduledTasks.contains(0)); + Assert.assertTrue(scheduledTasks.contains(1)); Assert.assertEquals(4, manager.numBipartiteSourceTasksCompleted); Assert.assertEquals(4, manager.numVertexManagerEventsReceived); Assert.assertEquals(Long.MAX_VALUE >> 1 << 1, manager.completedSourceTasksOutputSize); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java index 686320242e..f491df69f6 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java @@ -423,8 +423,8 @@ public void testAutoParallelism() throws Exception { // TODO improve tests for parallelism Assert.assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled Assert.assertEquals(2, scheduledTasks.size()); - Assert.assertTrue(scheduledTasks.contains(new Integer(0))); - Assert.assertTrue(scheduledTasks.contains(new Integer(1))); + Assert.assertTrue(scheduledTasks.contains(0)); + Assert.assertTrue(scheduledTasks.contains(1)); Assert.assertEquals(2, manager.numBipartiteSourceTasksCompleted); Assert.assertEquals(2, manager.numVertexManagerEventsReceived); Assert.assertEquals(100 * MB, manager.completedSourceTasksOutputSize); diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java index 9a4532eaae..4916a8d97a 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java @@ -75,7 +75,7 @@ public class TestInput extends AbstractLogicalInput { Set failingTaskIndices = Sets.newHashSet(); Set failingTaskAttempts = Sets.newHashSet(); Set failingInputIndices = Sets.newHashSet(); - Integer failAll = new Integer(-1); + Integer failAll = -1; int[] inputValues; /** @@ -227,7 +227,7 @@ public int doRead() { previousAttemptReadFailed = true; } else { for (int i=0 ; i failingTaskIndices = Sets.newHashSet(); int failingTaskAttemptUpto = 0; - Integer failAll = new Integer(-1); + Integer failAll = -1; int verifyValue = -1; Set verifyTaskIndices = Sets.newHashSet(); @@ -293,7 +293,7 @@ public void run(Map inputs, + " attempt: " + getContext().getTaskAttemptNumber() + " is: " + sum); if (verifyTaskIndices - .contains(new Integer(getContext().getTaskIndex()))) { + .contains(getContext().getTaskIndex())) { if (verifyValue != -1 && verifyValue != sum) { // expected output value set and not equal to observed value String msg = "Expected output mismatch of current FailingProcessor: " From e75e40003a05606b40d99f912d343dd8ced29b81 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Mon, 29 Jun 2026 13:43:57 +0530 Subject: [PATCH 3/6] Review comments Fix:1 --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 332084a724..4cfae4b17e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ org.apache apache - 37 + 38 org.apache.tez tez @@ -49,7 +49,6 @@ 25 25 - 3.6.2 true false UTF-8 From 45bde0455f66707e156b7fb0dbc5a0c79e52922d Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Mon, 29 Jun 2026 13:57:57 +0530 Subject: [PATCH 4/6] spotbugs fix --- pom.xml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pom.xml b/pom.xml index 4cfae4b17e..bd83a2e1f7 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ 3.2.0 4.9.3 4.9.3.0 + 9.8 2.3.0 1.15.0 33.4.8-jre @@ -1155,6 +1156,32 @@ plexus-velocity ${plexus-velocity.version} + + + org.ow2.asm + asm + ${asm.version} + + + org.ow2.asm + asm-commons + ${asm.version} + + + org.ow2.asm + asm-tree + ${asm.version} + + + org.ow2.asm + asm-analysis + ${asm.version} + + + org.ow2.asm + asm-util + ${asm.version} + ${basedir}/findbugs-exclude.xml From e9befdff706b4dd0caefd9d10ad1000b5338eec0 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Mon, 29 Jun 2026 14:03:01 +0530 Subject: [PATCH 5/6] Fix checkstyle errors --- .../common/objectregistry/TestObjectRegistry.java | 12 ++++++------ .../src/test/java/org/apache/tez/test/TestInput.java | 10 +++++++++- .../test/java/org/apache/tez/test/TestProcessor.java | 10 +++++++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java index cdcb20b81b..5313d9f6a7 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java @@ -31,14 +31,14 @@ private void testCRUD(ObjectRegistry objectRegistry) { Assert.assertNull(objectRegistry.get("foo")); Assert.assertFalse(objectRegistry.delete("foo")); Integer one = 1; - Integer two_1 = 2; - Integer two_2 = 3; + Integer twoFirst = 2; + Integer twoSecond = 3; Assert.assertNull(objectRegistry.cacheForDAG("one", one)); Assert.assertEquals(one, objectRegistry.get("one")); - Assert.assertNull(objectRegistry.cacheForDAG("two", two_1)); - Assert.assertNotNull(objectRegistry.cacheForSession("two", two_2)); - Assert.assertNotEquals(two_1, objectRegistry.get("two")); - Assert.assertEquals(two_2, objectRegistry.get("two")); + Assert.assertNull(objectRegistry.cacheForDAG("two", twoFirst)); + Assert.assertNotNull(objectRegistry.cacheForSession("two", twoSecond)); + Assert.assertNotEquals(twoFirst, objectRegistry.get("two")); + Assert.assertEquals(twoSecond, objectRegistry.get("two")); Assert.assertTrue(objectRegistry.delete("one")); Assert.assertFalse(objectRegistry.delete("one")); diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java index 4916a8d97a..52b287ca50 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java @@ -75,7 +75,15 @@ public class TestInput extends AbstractLogicalInput { Set failingTaskIndices = Sets.newHashSet(); Set failingTaskAttempts = Sets.newHashSet(); Set failingInputIndices = Sets.newHashSet(); - Integer failAll = -1; + private Integer failAll = -1; + + public Integer getFailAll() { + return failAll; + } + + public void setFailAll(Integer failAll) { + this.failAll = failAll; + } int[] inputValues; /** diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java index 7b2d8e4fea..e1fa7ad3cb 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java @@ -59,7 +59,15 @@ public class TestProcessor extends AbstractLogicalIOProcessor { long sleepMs; Set failingTaskIndices = Sets.newHashSet(); int failingTaskAttemptUpto = 0; - Integer failAll = -1; + private Integer failAll = -1; + + public Integer getFailAll() { + return failAll; + } + + public void setFailAll(Integer failAll) { + this.failAll = failAll; + } int verifyValue = -1; Set verifyTaskIndices = Sets.newHashSet(); From 2649a3c51e2a69cbf8ea8f3317ee32988807a277 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Mon, 29 Jun 2026 17:27:55 +0530 Subject: [PATCH 6/6] Review comments fix:2 --- README.md | 4 ++-- build-tools/docker/Dockerfile | 2 +- pom.xml | 31 ++----------------------------- tez-dist/src/docker/Dockerfile | 2 +- 4 files changed, 6 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 422c560783..2fc75d717d 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ For instructions on how to contribute to Tez, refer to: Requirements ------------ -* JDK 21+ +* JDK 25+ * Maven 3.9.14 or later -* spotbugs 4.9.3 or later (if running spotbugs) +* spotbugs 4.10.2 or later (if running spotbugs) * ProtocolBuffer 3.25.5 * Hadoop 3.x diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 63bee8dba2..ab0201357b 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -35,7 +35,7 @@ ARG HADOLINT_VERSION="2.14.0" ARG HADOLINT_SHA="5ffd7ed8f27894941a82f06229ed0dc75814eeb985d224d4fc3c7cf516f31cc7e6cc2d57348d4026084ac622c765e63c8274fdc6c36c0de03c4a5dda8f4ebf6f" ARG BUF_VERSION="1.68.2" ARG BUF_SHA="557ea42d00458466e3421bd1cf5781d882a95b0c1c0e54efffc326fdf9993d02" -ARG SPOTBUGS_VERSION="4.9.3" +ARG SPOTBUGS_VERSION="4.10.2" ARG MAVEN_VERSION="3.9.15" ARG ASTROID_VERSION="4.0.3" ARG PYLINT_VERSION="4.0.4" diff --git a/pom.xml b/pom.xml index bd83a2e1f7..2948481a76 100644 --- a/pom.xml +++ b/pom.xml @@ -76,9 +76,8 @@ 3.19.0 5.9.0 3.2.0 - 4.9.3 - 4.9.3.0 - 9.8 + 4.10.2 + 4.10.2.0 2.3.0 1.15.0 33.4.8-jre @@ -1156,32 +1155,6 @@ plexus-velocity ${plexus-velocity.version} - - - org.ow2.asm - asm - ${asm.version} - - - org.ow2.asm - asm-commons - ${asm.version} - - - org.ow2.asm - asm-tree - ${asm.version} - - - org.ow2.asm - asm-analysis - ${asm.version} - - - org.ow2.asm - asm-util - ${asm.version} - ${basedir}/findbugs-exclude.xml diff --git a/tez-dist/src/docker/Dockerfile b/tez-dist/src/docker/Dockerfile index 1db272e273..7b18b0f9da 100644 --- a/tez-dist/src/docker/Dockerfile +++ b/tez-dist/src/docker/Dockerfile @@ -34,7 +34,7 @@ RUN mkdir -p /opt/tez \ -C /opt/tez \ && rm -rf /opt/tez-$TEZ_VERSION.tar.gz -FROM eclipse-temurin:25-jdk-ubi10-minimal AS run +FROM eclipse-temurin:25-jdk-ubi9-minimal AS run ARG UID=1000 ARG TEZ_VERSION