Skip to content

Commit d41d9fa

Browse files
authored
fix(spark): restore version 3.5.7 (#1423)
* fix(spark): restore version 3.5.7 * update changelog * fix hbase-connectors patchable config
1 parent cd44e0a commit d41d9fa

11 files changed

Lines changed: 230 additions & 8 deletions

File tree

CHANGELOG.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ All notable changes to this project will be documented in this file.
5656
- trino: Remove `451` and `476` ([#1403]).
5757
- hbase: Remove `2.6.2` ([#1408]).
5858
- spark: remove the JRE before installing the JDK to prevent version conflicts ([#1410]).
59-
- spark: Remove `3.5.7` ([#1414]).
60-
- spark-connect-client: Remove `3.5.7` ([#1414]).
59+
- spark: Remove `3.5.7` in ([#1414]) but restored in ([#1423]).
60+
- spark-connect-client: Remove `3.5.7` in ([#1414]) but restored in ([#1423]).
6161

6262
### Fixed
6363

@@ -94,6 +94,7 @@ All notable changes to this project will be documented in this file.
9494
[#1409]: https://github.com/stackabletech/docker-images/pull/1409
9595
[#1410]: https://github.com/stackabletech/docker-images/pull/1410
9696
[#1414]: https://github.com/stackabletech/docker-images/pull/1414
97+
[#1423]: https://github.com/stackabletech/docker-images/pull/1414
9798
[#1422]: https://github.com/stackabletech/docker-images/pull/1422
9899

99100
## [25.11.0] - 2025-11-07

spark-connect-client/boil-config.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
[versions."3.5.7".local-images]
2+
spark-k8s = "3.5.7"
3+
java-base = "17"
4+
5+
[versions."3.5.7".build-arguments]
6+
python-version = "3.11"
7+
18
[versions."3.5.8".local-images]
29
spark-k8s = "3.5.8"
310
java-base = "17"

spark-k8s/boil-config.toml

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,25 @@
1+
[versions."3.5.7"]
2+
containerfile = "Dockerfile.3"
3+
4+
[versions."3.5.7".local-images]
5+
"hadoop/hadoop" = "3.4.2"
6+
java-base = "17"
7+
java-devel = "17"
8+
hbase = "2.6.3"
9+
"spark-k8s/hbase-connectors" = "1.0.1_3.5.7"
10+
11+
[versions."3.5.7".build-arguments]
12+
python-version = "3.11"
13+
aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop
14+
azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop
15+
azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop
16+
jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7
17+
stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
18+
woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
19+
jmx-exporter-version = "1.3.0"
20+
tini-version = "0.19.0"
21+
hbase-connector-version = "1.0.1_3.5.7"
22+
123
[versions."3.5.8"]
224
containerfile = "Dockerfile.3"
325

@@ -6,7 +28,7 @@ containerfile = "Dockerfile.3"
628
java-base = "17"
729
java-devel = "17"
830
hbase = "2.6.3"
9-
"spark-k8s/hbase-connectors" = "1.0.1_3"
31+
"spark-k8s/hbase-connectors" = "1.0.1_3.5.8"
1032

1133
[versions."3.5.8".build-arguments]
1234
python-version = "3.11"
@@ -18,7 +40,7 @@ stax2-api-version = "4.2.1" # needs to match the jackson version h
1840
woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
1941
jmx-exporter-version = "1.3.0"
2042
tini-version = "0.19.0"
21-
hbase-connector-version = "1.0.1_3"
43+
hbase-connector-version = "1.0.1_3.5.8"
2244

2345
[versions."4.0.1"]
2446
containerfile = "Dockerfile.4"
Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
1-
[versions."1.0.1_3"]
2-
containerfile = "Dockerfile"
31

4-
[versions."1.0.1_3".local-images]
2+
[versions."1.0.1_3.5.7".local-images]
53
java-devel = "17"
64

7-
[versions."1.0.1_3".build-arguments]
5+
[versions."1.0.1_3.5.7".build-arguments]
6+
spark-version = "3.5.7"
7+
hadoop-version = "3.4.2"
8+
hbase-version = "2.6.3"
9+
10+
[versions."1.0.1_3.5.8".local-images]
11+
java-devel = "17"
12+
13+
[versions."1.0.1_3.5.8".build-arguments]
814
spark-version = "3.5.8"
915
hadoop-version = "3.4.2"
1016
hbase-version = "2.6.3"

spark-k8s/hbase-connectors/stackable/patches/1.0.1_3/0001-Fix-protobuf-on-aarch64.patch renamed to spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/0001-Fix-protobuf-on-aarch64.patch

File renamed without changes.

spark-k8s/hbase-connectors/stackable/patches/1.0.1_3/patchable.toml renamed to spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/patchable.toml

File renamed without changes.
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
From 8211177df0ca6cae00bc3bebcd264364732691b3 Mon Sep 17 00:00:00 2001
2+
From: dervoeti <lukas.krug@stackable.tech>
3+
Date: Thu, 13 Mar 2025 22:10:56 +0100
4+
Subject: Fix protobuf on aarch64
5+
6+
---
7+
pom.xml | 19 ++++++++++++++++++-
8+
spark/pom.xml | 2 +-
9+
2 files changed, 19 insertions(+), 2 deletions(-)
10+
11+
diff --git a/pom.xml b/pom.xml
12+
index e849cd1..f514e14 100644
13+
--- a/pom.xml
14+
+++ b/pom.xml
15+
@@ -157,7 +157,10 @@
16+
<extra.enforcer.version>1.5.1</extra.enforcer.version>
17+
<restrict-imports.enforcer.version>0.14.0</restrict-imports.enforcer.version>
18+
<!--Internally we use a different version of protobuf. See hbase-protocol-shaded-->
19+
- <external.protobuf.version>2.5.0</external.protobuf.version>
20+
+ <!-- com.google repo will be used except on Aarch64 platform. -->
21+
+ <external.protobuf.groupId>com.google.protobuf</external.protobuf.groupId>
22+
+ <external.protobuf.version>2.6.1</external.protobuf.version>
23+
+ <external.protobuf.exe.version>${external.protobuf.version}</external.protobuf.exe.version>
24+
<protobuf.plugin.version>0.5.0</protobuf.plugin.version>
25+
<commons-io.version>2.11.0</commons-io.version>
26+
<avro.version>1.7.7</avro.version>
27+
@@ -933,5 +936,19 @@
28+
</plugins>
29+
</build>
30+
</profile>
31+
+ <!-- use com.github.os72 on aarch64 platform -->
32+
+ <profile>
33+
+ <id>aarch64</id>
34+
+ <properties>
35+
+ <external.protobuf.groupId>com.github.os72</external.protobuf.groupId>
36+
+ <external.protobuf.exe.version>2.6.1-build3</external.protobuf.exe.version>
37+
+ </properties>
38+
+ <activation>
39+
+ <os>
40+
+ <family>linux</family>
41+
+ <arch>aarch64</arch>
42+
+ </os>
43+
+ </activation>
44+
+ </profile>
45+
</profiles>
46+
</project>
47+
diff --git a/spark/pom.xml b/spark/pom.xml
48+
index 3f1eb21..fcdc73e 100644
49+
--- a/spark/pom.xml
50+
+++ b/spark/pom.xml
51+
@@ -84,7 +84,7 @@
52+
<artifactId>protobuf-maven-plugin</artifactId>
53+
<version>${protobuf.plugin.version}</version>
54+
<configuration>
55+
- <protocArtifact>com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
56+
+ <protocArtifact>${external.protobuf.groupId}:protoc:${external.protobuf.exe.version}:exe:${os.detected.classifier}</protocArtifact>
57+
<protoSourceRoot>${basedir}/src/main/protobuf/</protoSourceRoot>
58+
<clearOutputDirectory>false</clearOutputDirectory>
59+
<checkStaleness>true</checkStaleness>
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
base = "e5217d13ed729703580ff2d1b02378ada2d94f4d"
2+
mirror = "https://github.com/stackabletech/hbase-connectors.git"
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
From 5e3c5568d4bf0fdd4feb19e559d606a55a0d2569 Mon Sep 17 00:00:00 2001
2+
From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
3+
Date: Tue, 11 Mar 2025 17:29:39 +0200
4+
Subject: Update CycloneDX plugin
5+
6+
---
7+
dev/make-distribution.sh | 1 -
8+
pom.xml | 7 ++++++-
9+
2 files changed, 6 insertions(+), 2 deletions(-)
10+
11+
diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
12+
index ef7c010e93..0f4c1c74e4 100755
13+
--- a/dev/make-distribution.sh
14+
+++ b/dev/make-distribution.sh
15+
@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \
16+
-Dmaven.javadoc.skip=true \
17+
-Dmaven.scaladoc.skip=true \
18+
-Dmaven.source.skip \
19+
- -Dcyclonedx.skip=true \
20+
$@)
21+
22+
# Actually build the jar
23+
diff --git a/pom.xml b/pom.xml
24+
index a0e25ce4d8..b3755a04e7 100644
25+
--- a/pom.xml
26+
+++ b/pom.xml
27+
@@ -3534,7 +3534,12 @@
28+
<plugin>
29+
<groupId>org.cyclonedx</groupId>
30+
<artifactId>cyclonedx-maven-plugin</artifactId>
31+
- <version>2.7.9</version>
32+
+ <version>2.8.0</version>
33+
+ <configuration>
34+
+ <projectType>application</projectType>
35+
+ <schemaVersion>1.5</schemaVersion>
36+
+ <skipNotDeployed>false</skipNotDeployed>
37+
+ </configuration>
38+
<executions>
39+
<execution>
40+
<phase>package</phase>
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
From 0cfb1d6b733579ea91df842e685df9db5512ece8 Mon Sep 17 00:00:00 2001
2+
From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
3+
Date: Tue, 22 Jul 2025 17:34:03 +0200
4+
Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope
5+
6+
---
7+
LICENSE-binary | 1 +
8+
assembly/pom.xml | 14 ++++++++++++++
9+
dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 +
10+
licenses-binary/LICENSE-bouncycastle.txt | 13 +++++++++++++
11+
4 files changed, 29 insertions(+)
12+
create mode 100644 licenses-binary/LICENSE-bouncycastle.txt
13+
14+
diff --git a/LICENSE-binary b/LICENSE-binary
15+
index 05645977a0..9834cf333f 100644
16+
--- a/LICENSE-binary
17+
+++ b/LICENSE-binary
18+
@@ -480,6 +480,7 @@ org.typelevel:algebra_2.12:jar
19+
org.typelevel:cats-kernel_2.12
20+
org.typelevel:machinist_2.12
21+
net.razorvine:pickle
22+
+org.bouncycastle:bcprov-jdk18on
23+
org.slf4j:jcl-over-slf4j
24+
org.slf4j:jul-to-slf4j
25+
org.slf4j:slf4j-api
26+
diff --git a/assembly/pom.xml b/assembly/pom.xml
27+
index ffa6a0e2b0..19032d6054 100644
28+
--- a/assembly/pom.xml
29+
+++ b/assembly/pom.xml
30+
@@ -85,8 +85,22 @@
31+
<artifactId>guava</artifactId>
32+
<scope>${hadoop.deps.scope}</scope>
33+
</dependency>
34+
+
35+
+ <!--
36+
+ Adapted from https://github.com/apache/spark/pull/50077
37+
+
38+
+ SPARK-51311: HDFS-15098 (3.4.0) adds hard dependency on bcprov-jdk18on, Spark fails to submit
39+
+ to Kerberized cluster without this dependency, until HADOOP-19152 (3.5.0, unreleased)
40+
+ -->
41+
+ <dependency>
42+
+ <groupId>org.bouncycastle</groupId>
43+
+ <artifactId>bcprov-jdk18on</artifactId>
44+
+ <scope>${hadoop.deps.scope}</scope>
45+
+ </dependency>
46+
+
47+
</dependencies>
48+
49+
+
50+
<build>
51+
<plugins>
52+
<plugin>
53+
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
54+
index f110a1988f..88324d3e27 100644
55+
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
56+
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
57+
@@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar
58+
azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
59+
azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
60+
azure-storage/7.0.1//azure-storage-7.0.1.jar
61+
+bcprov-jdk18on/1.77//bcprov-jdk18on-1.77.jar
62+
blas/3.0.3//blas-3.0.3.jar
63+
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
64+
breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar
65+
diff --git a/licenses-binary/LICENSE-bouncycastle.txt b/licenses-binary/LICENSE-bouncycastle.txt
66+
new file mode 100644
67+
index 0000000000..277dcd1ebb
68+
--- /dev/null
69+
+++ b/licenses-binary/LICENSE-bouncycastle.txt
70+
@@ -0,0 +1,13 @@
71+
+Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org).
72+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
73+
+associated documentation files (the "Software"), to deal in the Software without restriction,
74+
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
75+
+sub license, and/or sell copies of the Software, and to permit persons to whom the Software is
76+
+furnished to do so, subject to the following conditions: The above copyright notice and this
77+
+permission notice shall be included in all copies or substantial portions of the Software.
78+
+
79+
+**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
80+
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
81+
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
82+
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
83+
+OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.**

0 commit comments

Comments
 (0)