diff --git a/CHANGELOG.md b/CHANGELOG.md index 8048100b5..7e830ca32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ All notable changes to this project will be documented in this file. - nifi: Remove `1.28.1`, deprecate `2.7.2` ([#1520]). - opa: Remove `1.8.0` ([#1509]). +- spark-k8s: Remove `3.5.7` and `4.0.1`, deprecate `3.5.8`, LTS `4.1.1` ([#1525]). [#1446]: https://github.com/stackabletech/docker-images/pull/1446 [#1452]: https://github.com/stackabletech/docker-images/pull/1452 @@ -61,6 +62,7 @@ All notable changes to this project will be documented in this file. [#1518]: https://github.com/stackabletech/docker-images/pull/1518 [#1520]: https://github.com/stackabletech/docker-images/pull/1520 [#1521]: https://github.com/stackabletech/docker-images/pull/1521 +[#1525]: https://github.com/stackabletech/docker-images/pull/1525 ## [26.3.0] - 2026-03-16 diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 1ebde375c..9ff8cf2ee 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -1,28 +1,7 @@ [metadata.registries] "oci.stackable.tech" = { namespace = "sdp" } -[versions."3.5.7"] -containerfile = "Dockerfile.3" - -[versions."3.5.7".local-images] -"hadoop/hadoop" = "3.4.2" -java-base = "17" -java-devel = "17" -hbase = "2.6.3" -"spark-k8s/hbase-connectors" = "1.0.1_3.5.7" - -[versions."3.5.7".build-arguments] -python-version = "3.11" -aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop -azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop -azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 -stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -jmx-exporter-version = "1.3.0" -tini-version = "0.19.0" -hbase-connector-version = "1.0.1_3.5.7" - +# Deprecated since 26.7.0 [versions."3.5.8"] containerfile = "Dockerfile.3" @@ -45,25 +24,8 @@ jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1_3.5.8" -[versions."4.0.1"] -containerfile = "Dockerfile.4" - -[versions."4.0.1".local-images] -"hadoop/hadoop" = "3.4.2" -java-base = "21" -java-devel = "21" - -[versions."4.0.1".build-arguments] -python-version = "3.12" -aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop -azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop -azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.18.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/4.0.1 -stax2-api-version = "4.2.2" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2 -woodstox-core-version = "7.0.0" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2 -jmx-exporter-version = "1.3.0" -tini-version = "0.19.0" +# LTS since 26.7.0 [versions."4.1.1"] containerfile = "Dockerfile.4" diff --git a/spark-k8s/hbase-connectors/boil-config.toml b/spark-k8s/hbase-connectors/boil-config.toml index 6cf4c0791..ae21e4cb6 100644 --- a/spark-k8s/hbase-connectors/boil-config.toml +++ b/spark-k8s/hbase-connectors/boil-config.toml @@ -1,12 +1,3 @@ - -[versions."1.0.1_3.5.7".local-images] -java-devel = "17" - -[versions."1.0.1_3.5.7".build-arguments] -spark-version = "3.5.7" -hadoop-version = "3.4.2" -hbase-version = "2.6.3" - [versions."1.0.1_3.5.8".local-images] java-devel = "17" diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/0001-Fix-protobuf-on-aarch64.patch b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/0001-Fix-protobuf-on-aarch64.patch deleted file mode 100644 index 37d8663c9..000000000 --- a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/0001-Fix-protobuf-on-aarch64.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 8211177df0ca6cae00bc3bebcd264364732691b3 Mon Sep 17 00:00:00 2001 -From: dervoeti -Date: Thu, 13 Mar 2025 22:10:56 +0100 -Subject: Fix protobuf on aarch64 - ---- - pom.xml | 19 ++++++++++++++++++- - spark/pom.xml | 2 +- - 2 files changed, 19 insertions(+), 2 deletions(-) - -diff --git a/pom.xml b/pom.xml -index e849cd1..f514e14 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -157,7 +157,10 @@ - 1.5.1 - 0.14.0 - -- 2.5.0 -+ -+ com.google.protobuf -+ 2.6.1 -+ ${external.protobuf.version} - 0.5.0 - 2.11.0 - 1.7.7 -@@ -933,5 +936,19 @@ - - - -+ -+ -+ aarch64 -+ -+ com.github.os72 -+ 2.6.1-build3 -+ -+ -+ -+ linux -+ aarch64 -+ -+ -+ - - -diff --git a/spark/pom.xml b/spark/pom.xml -index 3f1eb21..fcdc73e 100644 ---- a/spark/pom.xml -+++ b/spark/pom.xml -@@ -84,7 +84,7 @@ - protobuf-maven-plugin - ${protobuf.plugin.version} - -- com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier} -+ ${external.protobuf.groupId}:protoc:${external.protobuf.exe.version}:exe:${os.detected.classifier} - ${basedir}/src/main/protobuf/ - false - true diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/patchable.toml deleted file mode 100644 index 6d8c2c8db..000000000 --- a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -base = "e5217d13ed729703580ff2d1b02378ada2d94f4d" -mirror = "https://github.com/stackabletech/hbase-connectors.git"