diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 816d9f7be90..efeb04ce1e1 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -40,7 +40,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -90,7 +90,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
env:
INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell'
steps:
@@ -136,7 +136,7 @@ jobs:
fail-fast: false
matrix:
python: [ 3.9 ]
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -181,7 +181,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
# user/password => root/root
- name: Start mysql
@@ -232,10 +232,12 @@ jobs:
fail-fast: false
matrix:
include:
- - python: 3.9
+ - java: 11
+ python: 3.9
flink: 119
flink-profile: "1.19"
- - python: 3.9
+ - java: 17
+ python: 3.9
flink: 120
flink-profile: "1.20"
steps:
@@ -243,11 +245,11 @@ jobs:
uses: actions/checkout@v5
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- - name: Set up JDK 11
+ - name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v5
with:
distribution: 'temurin'
- java-version: 11
+ java-version: ${{ matrix.java }}
- name: Cache local Maven repository
uses: actions/cache@v5
with:
@@ -285,7 +287,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -366,22 +368,18 @@ jobs:
auto-activate: false
use-mamba: true
- name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
@@ -442,7 +440,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -472,7 +470,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
diff --git a/bin/common.cmd b/bin/common.cmd
index 0ddbad8a512..4931a081808 100644
--- a/bin/common.cmd
+++ b/bin/common.cmd
@@ -71,14 +71,38 @@ if not defined ZEPPELIN_JAVA_OPTS (
set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
)
+REM JPMS (Java Platform Module System) args mirrored from pom.xml extraJavaTestArgs.
+REM Targets JDK 17+: --add-modules, --enable-native-access, --sun-misc-unsafe-memory-access
+REM require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX flags.
+set JPMS_JAVA_OPTS=-XX:+IgnoreUnrecognizedVMOptions
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.io=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.net=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.nio=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.security.action=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.tryReflectionSetAccessible=true
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.allocator.type=pooled
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED
+
if not defined JAVA_OPTS (
set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
) else (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
+set JAVA_OPTS=%JAVA_OPTS% %JPMS_JAVA_OPTS%
-set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%
+set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %JPMS_JAVA_OPTS%
if not defined JAVA_HOME (
set ZEPPELIN_RUNNER=java
diff --git a/bin/common.sh b/bin/common.sh
index 31ff76c8a69..397115a674a 100644
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -147,7 +147,29 @@ if [[ ( -z "${ZEPPELIN_INTP_MEM}" ) && ( "${ZEPPELIN_INTERPRETER_LAUNCHER}" != "
export ZEPPELIN_INTP_MEM="-Xmx1024m"
fi
-JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}"
+# JPMS (Java Platform Module System) args mirrored from pom.xml extraJavaTestArgs.
+# Targets JDK 17+: --add-modules, --enable-native-access, --sun-misc-unsafe-memory-access
+# require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX flags.
+JPMS_JAVA_OPTS="-XX:+IgnoreUnrecognizedVMOptions"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.cs=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.security.action=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" -Dio.netty.tryReflectionSetAccessible=true"
+JPMS_JAVA_OPTS+=" -Dio.netty.allocator.type=pooled"
+JPMS_JAVA_OPTS+=" -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE"
+JPMS_JAVA_OPTS+=" --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED"
+JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM} ${JPMS_JAVA_OPTS}"
if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties"
else
@@ -155,7 +177,7 @@ else
fi
export JAVA_OPTS
-JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
+JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${JPMS_JAVA_OPTS}"
if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties -Dlog4j.configurationFile=file://${ZEPPELIN_CONF_DIR}/log4j2_docker.properties"
elif [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
diff --git a/flink/flink-scala-2.12/pom.xml b/flink/flink-scala-2.12/pom.xml
index 7d551123dd7..cde7bd45914 100644
--- a/flink/flink-scala-2.12/pom.xml
+++ b/flink/flink-scala-2.12/pom.xml
@@ -34,7 +34,6 @@
${flink1.19.version}
- 2.12.7
2.12
${flink.scala.version}
2.3.7
@@ -1060,7 +1059,7 @@
- -Xmx5120m -XX:MaxMetaspaceSize=1024m -Dsun.zip.disableMemoryMapping=true
+ -Xmx5120m -XX:MaxMetaspaceSize=1024m -Dsun.zip.disableMemoryMapping=true ${extraJavaTestArgs}
@@ -1222,6 +1221,38 @@
provided
+
+
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+
+
+ patch-flink-scala
+ initialize
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1256,6 +1287,35 @@
provided
+
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+
+
+ patch-flink-scala
+ initialize
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/flink/pom.xml b/flink/pom.xml
index 490ee787baa..19f8ffd1517 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -44,7 +44,7 @@
1.19.3
1.20.4
- 2.12.7
+ 2.12.20
2.12
diff --git a/pom.xml b/pom.xml
index a8e8a7f8f88..19b13ba4760 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,6 +178,29 @@
1.21.4
512m
+
+ -XX:+IgnoreUnrecognizedVMOptions
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+ --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+ --add-opens=java.base/java.io=ALL-UNNAMED
+ --add-opens=java.base/java.net=ALL-UNNAMED
+ --add-opens=java.base/java.nio=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+ --add-opens=java.base/sun.security.action=ALL-UNNAMED
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+ -Dio.netty.tryReflectionSetAccessible=true
+ -Dio.netty.allocator.type=pooled
+ -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
+ --sun-misc-unsafe-memory-access=allow
+ --enable-native-access=ALL-UNNAMED
+ -XX:+EnableDynamicAgentLoading
+
@@ -658,7 +681,7 @@
false
false
- -Xmx2g -Xms1g -Dfile.encoding=UTF-8
+ -Xmx2g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs}
true
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 0e276a49ba5..a51f045ae8e 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -55,8 +55,6 @@
**/PySparkInterpreterMatplotlibTest.java
**/*Test.*
-
-
@@ -453,36 +451,6 @@
-
-
- java-17
-
- [17,)
-
-
-
- -XX:+IgnoreUnrecognizedVMOptions
- --add-modules=jdk.incubator.vector
- --add-opens=java.base/java.lang=ALL-UNNAMED
- --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
- --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
- --add-opens=java.base/java.io=ALL-UNNAMED
- --add-opens=java.base/java.net=ALL-UNNAMED
- --add-opens=java.base/java.nio=ALL-UNNAMED
- --add-opens=java.base/java.util=ALL-UNNAMED
- --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
- --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
- --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
- --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
- --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
- --add-opens=java.base/sun.security.action=ALL-UNNAMED
- --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
- -Djdk.reflect.useDirectMethodHandle=false
- -Dio.netty.tryReflectionSetAccessible=true
-
-
-
-
diff --git a/zeppelin-integration/pom.xml b/zeppelin-integration/pom.xml
index fe665f2d01e..3368f7d0726 100644
--- a/zeppelin-integration/pom.xml
+++ b/zeppelin-integration/pom.xml
@@ -165,7 +165,7 @@
maven-surefire-plugin
- -Xmx2g -Xms1g -Dfile.encoding=UTF-8
+ -Xmx2g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs}
${tests.to.exclude}
diff --git a/zeppelin-interpreter-integration/pom.xml b/zeppelin-interpreter-integration/pom.xml
index a0a4e9ecd34..3197de27da3 100644
--- a/zeppelin-interpreter-integration/pom.xml
+++ b/zeppelin-interpreter-integration/pom.xml
@@ -174,7 +174,7 @@
1
false
- -Xmx3072m
+ -Xmx3072m ${extraJavaTestArgs}
${basedir}/../
diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml
index 8dbdb575673..505c5ed75bb 100644
--- a/zeppelin-server/pom.xml
+++ b/zeppelin-server/pom.xml
@@ -585,7 +585,7 @@
1
false
- -Xmx3g -Xms1g -Dfile.encoding=UTF-8
+ -Xmx3g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs}
${project.build.directory}
diff --git a/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java b/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
index 8a7dd6bdc4d..86b48c45310 100644
--- a/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
+++ b/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
@@ -455,6 +455,7 @@ public static String downloadFlink(String flinkVersion, String scalaVersion) {
if (targetFlinkHomeFolder.exists()) {
LOGGER.info("Skip to download Flink {}_{} as it is already downloaded.", flinkVersion,
scalaVersion);
+ patchFlinkScala(targetFlinkHomeFolder, flinkVersion, scalaVersion);
return targetFlinkHomeFolder.getAbsolutePath();
}
File flinkTGZ = new File(flinkDownloadFolder,
@@ -541,9 +542,52 @@ public static String downloadFlink(String flinkVersion, String scalaVersion) {
} catch (Exception e) {
throw new RuntimeException("Fail to download jar", e);
}
+ patchFlinkScala(targetFlinkHomeFolder, flinkVersion, scalaVersion);
return targetFlinkHomeFolder.getAbsolutePath();
}
+ /**
+ * Flink 1.x tgz bundles flink-scala_2.12 which shades scala 2.12.7.
+ * Scala 2.12.7's JrtClassPath.asURLs() crashes on JDK 17+ (scala/bug#11608, fixed in 2.12.9).
+ * Strip the shaded classes and copy in Scala 2.12.20 jars.
+ */
+ private static void patchFlinkScala(File flinkHome, String flinkVersion, String scalaVersion) {
+ if (!"2.12".equals(scalaVersion)) {
+ return;
+ }
+ if (SemanticVersion.of(flinkVersion).equalsOrNewerThan(SemanticVersion.of("2.0.0"))) {
+ return;
+ }
+ File libDir = new File(flinkHome, "lib");
+ for (File jar : libDir.listFiles((d, n) -> n.startsWith("flink-scala_") && n.endsWith(".jar"))) {
+ LOGGER.info("Stripping Scala 2.12.7 classes from {}", jar.getName());
+ try {
+ Process p = new ProcessBuilder("zip", "-d", jar.getAbsolutePath(), "scala/*")
+ .redirectErrorStream(true).start();
+ IOUtils.toString(p.getInputStream(), StandardCharsets.UTF_8);
+ p.waitFor();
+ } catch (Exception e) {
+ LOGGER.warn("Failed to strip Scala classes from {}", jar.getName(), e);
+ }
+ }
+ String scalaPatchVersion = "2.12.20";
+ for (String artifact : new String[]{"scala-library", "scala-compiler", "scala-reflect"}) {
+ String jarName = artifact + "-" + scalaPatchVersion + ".jar";
+ File dest = new File(libDir, jarName);
+ if (dest.exists()) {
+ continue;
+ }
+ String url = "https://repo1.maven.org/maven2/org/scala-lang/" + artifact + "/"
+ + scalaPatchVersion + "/" + jarName;
+ try {
+ LOGGER.info("Downloading {} to {}", jarName, dest);
+ download(url, 3, dest);
+ } catch (Exception e) {
+ throw new RuntimeException("Fail to download " + jarName, e);
+ }
+ }
+ }
+
private static void mvFile(String srcPath, String dstPath) throws IOException {
Path src = Paths.get(srcPath);
Path dst = Paths.get(dstPath);