diff --git a/CHANGELOG.md b/CHANGELOG.md index 9931707a..d4476f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ All notable changes to this project will be documented in this file. ### Removed +- Support for Spark `3.5.7` and `4.0.1` ([#695]) - Support for Spark `3.5.6` ([#642]). - Deprecated support for Spark `3.5.7` ([#650]). @@ -86,6 +87,7 @@ All notable changes to this project will be documented in this file. [#663]: https://github.com/stackabletech/spark-k8s-operator/pull/663 [#664]: https://github.com/stackabletech/spark-k8s-operator/pull/664 [#666]: https://github.com/stackabletech/spark-k8s-operator/pull/666 +[#695]: https://github.com/stackabletech/spark-k8s-operator/pull/695 ## [25.11.0] - 2025-11-07 diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index 8a5f78f4..17bbbe0b 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.8 + productVersion: 4.1.1 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index 8dbb61f1..235a47f1 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.8 + productVersion: 4.1.1 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index e12fa1fd..69ca47be 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.8 + productVersion: 4.1.1 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 4bbe3e0e..97d0fd68 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: - productVersion: 3.5.8 + productVersion: 4.1.1 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index 9759aa90..6d7d598f 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.8 + productVersion: 4.1.1 mode: cluster mainApplicationFile: s3a://my-bucket/app.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 3c360239..cadc56e2 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.8 + productVersion: 4.1.1 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index f3c8a5f3..c64799ec 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.8 + productVersion: 4.1.1 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/application.yaml b/docs/modules/spark-k8s/examples/getting_started/application.yaml index 5057495b..4c47f007 100644 --- a/docs/modules/spark-k8s/examples/getting_started/application.yaml +++ b/docs/modules/spark-k8s/examples/getting_started/application.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: # <2> - productVersion: 3.5.8 + productVersion: 4.1.1 mode: cluster # <3> mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py # <4> job: # <5> diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 72c53867..ed0e6b2c 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,13 +3,9 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 4.1.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) -- 4.0.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) (Deprecated) -- 3.5.8 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (LTS) -- 3.5.7 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (Deprecated) +- 4.1.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) (LTS) +- 3.5.8 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (Deprecated) -Apache Spark 4.1.1 has the following known issues (as of February 2026): +Apache Spark 4.1.1 has the following known issues (as of July 2026): -- Missing HBase compatibility (See: https://github.com/apache/hbase-connectors/pull/130) -- No Iceberg Spark runtime release with support for Spark 4.1 available yet. -- No Delta Lake release with support for Spark 4.1 available yet. +- Missing HBase compatibility (See: https://issues.apache.org/jira/browse/HBASE-30178) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 9e21539c..601bae0b 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,40 +5,29 @@ dimensions: - "false" - name: spark values: - - 3.5.7 - 3.5.8 - - 4.0.1 - 4.1.1 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-logging values: - - 3.5.7 - 3.5.8 - - 4.0.1 - 4.1.1 - name: spark-hbase-connector values: - - 3.5.7 - 3.5.8 # No hbase-connector release with support for Spark 4 yet. - # - 4.0.1 # - 4.1.1 - name: spark-delta-lake values: - - 3.5.7 - 3.5.8 - - 4.0.1 - # No delta-lake release with support for Spark 4.1 yet - # - 4.1.1 + - 4.1.1 # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev # Note: We keep this list, as new Spark versions are only supported delayed by Iceberg - name: spark-iceberg values: - - 3.5.7 - 3.5.8 - - 4.0.1 - 4.1.1 - name: iceberg-latest values: @@ -48,9 +37,7 @@ dimensions: - 4.0.0 - name: spark-connect values: - - 3.5.7 - 3.5.8 - - 4.0.1 - 4.1.1 # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: hbase