[ZEPPELIN-5966] Support Spark 3.5

apache · Oct 8, 2023 · 18d8a9e · 18d8a9e
1 parent 3ebb815
commit 18d8a9e
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 1 deletion.
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
@@ -411,6 +411,10 @@ jobs:
         run: |
           rm -rf spark/interpreter/metastore_db
           ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+      - name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }}
+        run: |
+          rm -rf spark/interpreter/metastore_db
+          ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
 
   livy-0-7-with-spark-3-4-1-under-python3:
     runs-on: ubuntu-20.04

diff --git a/docs/setup/basics/how_to_build.md b/docs/setup/basics/how_to_build.md
@@ -95,7 +95,7 @@ Set scala version (default 2.11). Available profiles are
 
 To be noticed, the spark profiles here only affect the unit test (no need to specify `SPARK_HOME`) of spark interpreter. 
 Zeppelin doesn't require you to build with different spark to make different versions of spark work in Zeppelin.
-You can run different versions of Spark in Zeppelin as long as you specify `SPARK_HOME`. Actually Zeppelin supports all the versions of Spark from 3.2 to 3.4.
+You can run different versions of Spark in Zeppelin as long as you specify `SPARK_HOME`. Actually Zeppelin supports all the versions of Spark from 3.2 to 3.5.
 
 To build with a specific Spark version or scala versions, define one or more of the following profiles and options:
 
@@ -106,6 +106,7 @@ Set spark major version
 Available profiles are
 
 ```
+-Pspark-3.5
 -Pspark-3.4
 -Pspark-3.3
 -Pspark-3.2

diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
@@ -511,6 +511,18 @@
     </profile>
 
     <!-- profile spark-x only affect spark version used in test -->
+    <profile>
+      <id>spark-3.5</id>
+      <properties>
+        <datanucleus.core.version>4.1.17</datanucleus.core.version>
+        <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version>
+        <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version>
+        <spark.version>3.5.0</spark.version>
+        <protobuf.version>3.21.12</protobuf.version>
+        <py4j.version>0.10.9.7</py4j.version>
+      </properties>
+    </profile>
+
     <profile>
       <id>spark-3.4</id>
       <activation>