[LIVY-1041] Add Spark 4 support via a new Maven profile

.github/workflows/integration-tests.yaml

@@ -28,11 +28,14 @@ jobs:
     container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest
     strategy:
       matrix:
-        maven_profile:
-        - "-Pscala-2.12 -Pspark3"
-        jdk_path:
-        - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
-        - "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+        include:
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+          # Spark 4 requires JDK 17+ and Scala 2.13; JDK 8 is unsupported.
+          - maven_profile: "-Pscala-2.13 -Pspark4"
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
     steps:
     -
       name: Checkout
@@ -48,8 +51,13 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
     -
+      # Must match the pyenv-installed version in dev/docker/livy-dev-base/Dockerfile.
+      # Per each Spark release's python/setup.py: Spark 4.1 supports 3.10..3.14
+      # (python_requires=">=3.10"), Spark 3.5 supports 3.8..3.11
+      # (python_requires=">=3.8"). The intersection is 3.10 and 3.11; 3.11.11
+      # is picked as the shared version.
       name: Set Python 3 as default
-      run: pyenv global 3 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV"
+      run: pyenv global 3.11.11 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV"
     -
       name: Set JDK version
       run: update-alternatives --set java ${{ matrix.jdk_path }}

.github/workflows/unit-tests.yaml

@@ -29,11 +29,14 @@ jobs:
     container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest
     strategy:
       matrix:
-        maven_profile:
-        - "-Pscala-2.12 -Pspark3"
-        jdk_path:
-        - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
-        - "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+        include:
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+          # Spark 4 requires JDK 17+ and Scala 2.13; JDK 8 is unsupported.
+          - maven_profile: "-Pscala-2.13 -Pspark4"
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
     steps:
     -
       name: Checkout

README.md

@@ -92,9 +92,21 @@ version of Spark without needing to rebuild.
 
 ### Build Profiles
 
-| Flag           | Purpose                                    |
-|----------------|--------------------------------------------|
-| -Phadoop2      | Choose Hadoop2 based build dependencies    |
-| -Pthriftserver | Build and test Livy Thrift Server modules  |
-| -Pspark3       | Choose Spark 3.x based build dependencies  |
-| -Pscala-2.12   | Choose Scala 2.12 based build dependencies |
+| Flag           | Purpose                                                                            |
+|----------------|------------------------------------------------------------------------------------|
+| -Phadoop2      | Choose Hadoop2 based build dependencies                                            |
+| -Pthriftserver | Build and test Livy Thrift Server modules                                          |
+| -Pspark3       | Choose Spark 3.x based build dependencies                                          |
+| -Pspark4       | Choose Spark 4.x (4.1.2) based build dependencies (requires JDK 17+ + Scala 2.13)  |
+| -Pscala-2.12   | Choose Scala 2.12 based build dependencies                                         |
+| -Pscala-2.13   | Choose Scala 2.13 based build dependencies (used with `-Pspark4`)                  |
+
+Example — build against Spark 4:
+
+```
+mvn package -Pspark4 -Pscala-2.13
+```
+
+> **Note**: `-Pspark4` requires JDK 17 or JDK 21 and pulls in Scala 2.13 and Hadoop 3.4.1.
+> JDK 8, JDK 11 and Scala 2.12 are not supported by Spark 4. Supported Python
+> versions for Spark 4.1 are 3.10 – 3.14.

client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala

@@ -30,7 +30,8 @@ import scala.concurrent.{ExecutionContext, Future}
 import org.mockito.ArgumentCaptor
 import org.mockito.Matchers.{eq => meq, _}
 import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.funspec.AnyFunSpecLike
 import org.scalatra.LifeCycle
 import org.scalatra.servlet.ScalatraListener
 
@@ -49,7 +50,7 @@ import org.apache.livy.utils.AppInfo
  * module, which implements the client session backend. The client servlet has some functionality
  * overridden to avoid creating sub-processes for each seession.
  */
-class HttpClientSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
+class HttpClientSpec extends AnyFunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
 
   import HttpClientSpec._
 

client-http/src/test/scala/org/apache/livy/client/http/LivyConnectionSpec.scala

@@ -25,15 +25,16 @@ import org.eclipse.jetty.security._
 import org.eclipse.jetty.security.UserStore
 import org.eclipse.jetty.security.authentication.BasicAuthenticator
 import org.eclipse.jetty.util.security._
-import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
-import org.scalatest.Matchers._
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.funspec.AnyFunSpecLike
+import org.scalatest.matchers.should.Matchers._
 import org.scalatra.servlet.ScalatraListener
 
 import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
 import org.apache.livy.client.common.TestUtils
 import org.apache.livy.server.WebServer
 
-class LivyConnectionSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
+class LivyConnectionSpec extends AnyFunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
   describe("LivyConnection") {
     def basicAuth(username: String, password: String, realm: String): SecurityHandler = {
       val roles = Array("user")

core/scala-2.13/pom.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.livy</groupId>
+  <artifactId>livy-core_2.13</artifactId>
+  <version>1.0.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <parent>
+    <groupId>org.apache.livy</groupId>
+    <artifactId>livy-core-parent</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

core/src/test/scala/org/apache/livy/EOLUtilsSuite.scala

@@ -17,9 +17,9 @@
 
 package org.apache.livy
 
-import org.scalatest.FunSuite
+import org.scalatest.funsuite.AnyFunSuite
 
-class EOLUtilsSuite extends FunSuite with LivyBaseUnitTestSuite {
+class EOLUtilsSuite extends AnyFunSuite with LivyBaseUnitTestSuite {
 
   test("check EOL") {
     val s1 = "test\r\ntest"

dev/docker/livy-dev-base/Dockerfile

@@ -70,8 +70,17 @@ RUN git clone https://github.com/pyenv/pyenv.git $HOME/pyenv
 ENV PYENV_ROOT=$HOME/pyenv
 ENV PATH="$HOME/pyenv/shims:$HOME/pyenv/bin:$HOME/bin:$PATH"
 
-RUN pyenv install -v 3.9.21 && \
-  pyenv global 3.9.21 && \
+# Python 3.11.11 is chosen because it is in the officially supported range of
+# BOTH matrix profiles, per each Spark release's python/setup.py:
+#   * -Pspark4 (Spark 4.1.2): python_requires=">=3.10", classifiers list
+#     3.10 / 3.11 / 3.12 / 3.13 / 3.14.
+#   * -Pspark3 (Spark 3.5.6): python_requires=">=3.8", classifiers list
+#     3.8 / 3.9 / 3.10 / 3.11.
+# The intersection is 3.10 and 3.11; we pick the higher one (3.11.11) so the
+# same image serves both profiles. When bumping this, keep
+# .github/workflows/integration-tests.yaml's `pyenv global` in sync.
+RUN pyenv install -v 3.11.11 && \
+  pyenv global 3.11.11 && \
   pyenv rehash
 
 # Install build dependencies for python3

integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala

@@ -38,9 +38,11 @@ import org.apache.http.client.params.AuthPolicy
 import org.apache.http.impl.auth.BasicSchemeFactory
 import org.apache.http.impl.auth.SPNegoSchemeFactory
 import org.apache.http.impl.client.DefaultHttpClient
-import org.scalatest._
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
 
-abstract class BaseIntegrationTestSuite extends FunSuite with Matchers with BeforeAndAfterAll {
+abstract class BaseIntegrationTestSuite extends AnyFunSuite with Matchers with BeforeAndAfterAll {
   import scala.concurrent.ExecutionContext.Implicits.global
 
   var cluster: Cluster = _

integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala

@@ -30,27 +30,46 @@ import org.apache.livy.rsc.RSCConf
 import org.apache.livy.sessions._
 import org.apache.livy.test.framework.{BaseIntegrationTestSuite, LivyRestClient}
 
-class InteractiveIT extends BaseIntegrationTestSuite {
+class InteractiveIT extends BaseIntegrationTestSuite with ScalaVersionAware {
   test("basic interactive session") {
     withNewSession(Spark) { s =>
       s.run("val sparkVersion = sc.version").result().left.foreach(info(_))
       s.run("val scalaVersion = util.Properties.versionString").result().left.foreach(info(_))
-      s.run("1+1").verifyResult("res0: Int = 2\n")
+      // Scala 2.13's REPL prints a `val ` prefix before result names
+      // (`val res0: Int = 2`) whereas Scala 2.12 prints just `res0: Int = 2`.
+      // Accept both forms so the test passes on either -Pscala-2.12 or
+      // -Pscala-2.13.
+      s.run("1+1").verifyResult(s"${optionalValPrefixRegex}res0: Int = 2\n")
 
       // Ignore the following line if running on a external cluster due to config differences
       // with the mini cluster
-      s.run("""sc.getConf.get("spark.executor.instances")""").verifyResult("res1: String = 1\n")
+      s.run("""sc.getConf.get("spark.executor.instances")""")
+        .verifyResult(s"${optionalValPrefixRegex}res1: String = 1\n")
 
+      // Spark 4 relocated SQLContext into the `org.apache.spark.sql.classic`
+      // package; Spark 3 keeps it directly under `org.apache.spark.sql`. Match
+      // either shape (`SQLContext` or `classic.SQLContext`) and tolerate the
+      // Scala 2.13 REPL's `val ` prefix as elsewhere in this test.
       s.run("val sql = spark.sqlContext").verifyResult(
-        ".*" + Pattern.quote(
-        "sql: org.apache.spark.sql.SQLContext = org.apache.spark.sql.SQLContext") + ".*")
-      s.run("abcde").verifyError(evalue = ".*?:[0-9]+: error: not found: value abcde.*")
+        ".*sql: org\\.apache\\.spark\\.sql\\.(?:classic\\.)?SQLContext = " +
+        "org\\.apache\\.spark\\.sql\\.(?:classic\\.)?SQLContext.*")
+      // Scala 2.12's REPL prefixes compile errors with a "<source>:<line>:"
+      // location marker (e.g. "<console>:12: error: not found: value abcde"),
+      // while Scala 2.13 drops the marker and just emits "error: not found:
+      // value abcde". Accept both.
+      s.run("abcde").verifyError(evalue = ".*(?:.*?:[0-9]+: )?error: not found: value abcde.*")
       s.run("throw new IllegalStateException()")
         .verifyError(evalue = ".*java\\.lang\\.IllegalStateException.*")
 
-      // Verify query submission
+      // Verify query submission. Spark 4's Scala 2.13 REPL surfaces the runtime
+      // class name (`org.apache.spark.sql.classic.DataFrame`) rather than the
+      // compile-time alias (`org.apache.spark.sql.DataFrame`) that Spark 3
+      // prints, and prefixes the identifier with `val ` like elsewhere in this
+      // test. Accept both shapes.
       s.run(s"""val df = spark.createDataFrame(Seq(("jerry", 20), ("michael", 21)))""")
-        .verifyResult(".*" + Pattern.quote("df: org.apache.spark.sql.DataFrame") + ".*")
+        .verifyResult(
+          s".*${optionalValPrefixRegex}df: " +
+            "org\\.apache\\.spark\\.sql\\.(?:classic\\.)?DataFrame.*")
       s.run("df.createOrReplaceTempView(\"people\")").result()
       s.run("SELECT * FROM people", Some(SQL)).verifyResult(".*\"jerry\",20.*\"michael\",21.*")
 
@@ -167,9 +186,15 @@ class InteractiveIT extends BaseIntegrationTestSuite {
       s.run("import org.codehaus.plexus.util._").verifyResult("import org.codehaus.plexus.util._\n")
 
       // Check does SparkContext see classes defined by Scala interpreter.
-      s.run("case class Item(i: Int)").verifyResult("defined class Item\n")
+      // Scala 2.12's REPL reports `defined class Item`; Scala 2.13 emits the
+      // shorter `class Item`. Accept both.
+      s.run("case class Item(i: Int)").verifyResult(s"${optionalDefinedPrefixRegex}class Item\n")
+      // Scala 2.13's REPL prefixes with `val ` (e.g. `val rdd: ...`); 2.12
+      // omits it. Spark 4 (Scala 2.13) also prints a deprecation warning
+      // before the value binding because `parallelize` is now deprecated on
+      // SparkContext -- accept an optional warning header.
       s.run("val rdd = sc.parallelize(Array.fill(10){new Item(scala.util.Random.nextInt(1000))})")
-        .verifyResult("rdd.*")
+        .verifyResult(s"(?s)${optionalWarningPrefixRegex}${optionalValPrefixRegex}rdd.*")
       s.run("rdd.count()").verifyResult(".*= 10\n")
     }
   }
@@ -188,15 +213,17 @@ class InteractiveIT extends BaseIntegrationTestSuite {
   test("recover interactive session") {
     withNewSession(Spark) { s =>
       val stmt1 = s.run("1")
-      stmt1.verifyResult("res0: Int = 1\n")
+      // Scala 2.13's REPL renders value results as `val res0: Int = 1`
+      // whereas Scala 2.12 prints `res0: Int = 1`; accept both.
+      stmt1.verifyResult(s"${optionalValPrefixRegex}res0: Int = 1\n")
 
       restartLivy()
 
       // Verify session still exists.
       s.verifySessionIdle()
-      s.run("2").verifyResult("res1: Int = 2\n")
+      s.run("2").verifyResult(s"${optionalValPrefixRegex}res1: Int = 2\n")
       // Verify statement result is preserved.
-      stmt1.verifyResult("res0: Int = 1\n")
+      stmt1.verifyResult(s"${optionalValPrefixRegex}res0: Int = 1\n")
 
       s.stop()