diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 816d9f7be90..efeb04ce1e1 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -40,7 +40,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11 ] + java: [ 11, 17 ] steps: - name: Checkout uses: actions/checkout@v5 @@ -90,7 +90,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11 ] + java: [ 11, 17 ] env: INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' steps: @@ -136,7 +136,7 @@ jobs: fail-fast: false matrix: python: [ 3.9 ] - java: [ 11 ] + java: [ 11, 17 ] steps: - name: Checkout uses: actions/checkout@v5 @@ -181,7 +181,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11 ] + java: [ 11, 17 ] steps: # user/password => root/root - name: Start mysql @@ -232,10 +232,12 @@ jobs: fail-fast: false matrix: include: - - python: 3.9 + - java: 11 + python: 3.9 flink: 119 flink-profile: "1.19" - - python: 3.9 + - java: 17 + python: 3.9 flink: 120 flink-profile: "1.20" steps: @@ -243,11 +245,11 @@ jobs: uses: actions/checkout@v5 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Set up JDK 11 + - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v5 with: distribution: 'temurin' - java-version: 11 + java-version: ${{ matrix.java }} - name: Cache local Maven repository uses: actions/cache@v5 with: @@ -285,7 +287,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11 ] + java: [ 11, 17 ] steps: - name: Checkout uses: actions/checkout@v5 @@ -366,22 +368,18 @@ jobs: auto-activate: false use-mamba: true - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} - if: ${{ matrix.java == 11 }} run: | rm -rf spark/interpreter/metastore_db ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }} - if: ${{ matrix.java == 11 }} run: | rm -rf spark/interpreter/metastore_db ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }} - if: ${{ matrix.java == 11 }} run: | rm -rf spark/interpreter/metastore_db ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }} - if: ${{ matrix.java == 11 }} run: | rm -rf spark/interpreter/metastore_db ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} @@ -442,7 +440,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11 ] + java: [ 11, 17 ] steps: - name: Checkout uses: actions/checkout@v5 @@ -472,7 +470,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11 ] + java: [ 17 ] steps: - name: Checkout uses: actions/checkout@v5 diff --git a/bin/common.cmd b/bin/common.cmd index 0ddbad8a512..4931a081808 100644 --- a/bin/common.cmd +++ b/bin/common.cmd @@ -71,14 +71,38 @@ if not defined ZEPPELIN_JAVA_OPTS ( set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM% ) +REM JPMS (Java Platform Module System) args mirrored from pom.xml extraJavaTestArgs. +REM Targets JDK 17+: --add-modules, --enable-native-access, --sun-misc-unsafe-memory-access +REM require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX flags. +set JPMS_JAVA_OPTS=-XX:+IgnoreUnrecognizedVMOptions +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang.invoke=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang.reflect=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.io=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.net=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.nio=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util.concurrent=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.nio.ch=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.nio.cs=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.security.action=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.util.calendar=ALL-UNNAMED +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.tryReflectionSetAccessible=true +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.allocator.type=pooled +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE +set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED + if not defined JAVA_OPTS ( set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% ) else ( set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS% ) +set JAVA_OPTS=%JAVA_OPTS% %JPMS_JAVA_OPTS% -set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% +set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %JPMS_JAVA_OPTS% if not defined JAVA_HOME ( set ZEPPELIN_RUNNER=java diff --git a/bin/common.sh b/bin/common.sh index 31ff76c8a69..397115a674a 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -147,7 +147,29 @@ if [[ ( -z "${ZEPPELIN_INTP_MEM}" ) && ( "${ZEPPELIN_INTERPRETER_LAUNCHER}" != " export ZEPPELIN_INTP_MEM="-Xmx1024m" fi -JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}" +# JPMS (Java Platform Module System) args mirrored from pom.xml extraJavaTestArgs. +# Targets JDK 17+: --add-modules, --enable-native-access, --sun-misc-unsafe-memory-access +# require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX flags. +JPMS_JAVA_OPTS="-XX:+IgnoreUnrecognizedVMOptions" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.cs=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.security.action=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED" +JPMS_JAVA_OPTS+=" -Dio.netty.tryReflectionSetAccessible=true" +JPMS_JAVA_OPTS+=" -Dio.netty.allocator.type=pooled" +JPMS_JAVA_OPTS+=" -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE" +JPMS_JAVA_OPTS+=" --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED" +JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM} ${JPMS_JAVA_OPTS}" if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties" else @@ -155,7 +177,7 @@ else fi export JAVA_OPTS -JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}" +JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${JPMS_JAVA_OPTS}" if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties -Dlog4j.configurationFile=file://${ZEPPELIN_CONF_DIR}/log4j2_docker.properties" elif [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then diff --git a/flink/flink-scala-2.12/pom.xml b/flink/flink-scala-2.12/pom.xml index 7d551123dd7..cde7bd45914 100644 --- a/flink/flink-scala-2.12/pom.xml +++ b/flink/flink-scala-2.12/pom.xml @@ -34,7 +34,6 @@ ${flink1.19.version} - 2.12.7 2.12 ${flink.scala.version} 2.3.7 @@ -1060,7 +1059,7 @@ - -Xmx5120m -XX:MaxMetaspaceSize=1024m -Dsun.zip.disableMemoryMapping=true + -Xmx5120m -XX:MaxMetaspaceSize=1024m -Dsun.zip.disableMemoryMapping=true ${extraJavaTestArgs} @@ -1222,6 +1221,38 @@ provided + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + patch-flink-scala + initialize + run + + + + + + + + + + + + + + + + + + + @@ -1256,6 +1287,35 @@ provided + + + + org.apache.maven.plugins + maven-antrun-plugin + + + patch-flink-scala + initialize + run + + + + + + + + + + + + + + + + + + + diff --git a/flink/pom.xml b/flink/pom.xml index 490ee787baa..19f8ffd1517 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -44,7 +44,7 @@ 1.19.3 1.20.4 - 2.12.7 + 2.12.20 2.12 diff --git a/pom.xml b/pom.xml index a8e8a7f8f88..19b13ba4760 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,29 @@ 1.21.4 512m + + -XX:+IgnoreUnrecognizedVMOptions + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + -Dio.netty.tryReflectionSetAccessible=true + -Dio.netty.allocator.type=pooled + -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE + --sun-misc-unsafe-memory-access=allow + --enable-native-access=ALL-UNNAMED + -XX:+EnableDynamicAgentLoading + @@ -658,7 +681,7 @@ false false - -Xmx2g -Xms1g -Dfile.encoding=UTF-8 + -Xmx2g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs} true diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index 0e276a49ba5..a51f045ae8e 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -55,8 +55,6 @@ **/PySparkInterpreterMatplotlibTest.java **/*Test.* - - @@ -453,36 +451,6 @@ - - - java-17 - - [17,) - - - - -XX:+IgnoreUnrecognizedVMOptions - --add-modules=jdk.incubator.vector - --add-opens=java.base/java.lang=ALL-UNNAMED - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED - --add-opens=java.base/java.lang.reflect=ALL-UNNAMED - --add-opens=java.base/java.io=ALL-UNNAMED - --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/java.nio=ALL-UNNAMED - --add-opens=java.base/java.util=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/sun.nio.cs=ALL-UNNAMED - --add-opens=java.base/sun.security.action=ALL-UNNAMED - --add-opens=java.base/sun.util.calendar=ALL-UNNAMED - -Djdk.reflect.useDirectMethodHandle=false - -Dio.netty.tryReflectionSetAccessible=true - - - - diff --git a/zeppelin-integration/pom.xml b/zeppelin-integration/pom.xml index fe665f2d01e..3368f7d0726 100644 --- a/zeppelin-integration/pom.xml +++ b/zeppelin-integration/pom.xml @@ -165,7 +165,7 @@ maven-surefire-plugin - -Xmx2g -Xms1g -Dfile.encoding=UTF-8 + -Xmx2g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs} ${tests.to.exclude} diff --git a/zeppelin-interpreter-integration/pom.xml b/zeppelin-interpreter-integration/pom.xml index a0a4e9ecd34..3197de27da3 100644 --- a/zeppelin-interpreter-integration/pom.xml +++ b/zeppelin-interpreter-integration/pom.xml @@ -174,7 +174,7 @@ 1 false - -Xmx3072m + -Xmx3072m ${extraJavaTestArgs} ${basedir}/../ diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 8dbdb575673..505c5ed75bb 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -585,7 +585,7 @@ 1 false - -Xmx3g -Xms1g -Dfile.encoding=UTF-8 + -Xmx3g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs} ${project.build.directory} diff --git a/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java b/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java index 8a7dd6bdc4d..86b48c45310 100644 --- a/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java +++ b/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java @@ -455,6 +455,7 @@ public static String downloadFlink(String flinkVersion, String scalaVersion) { if (targetFlinkHomeFolder.exists()) { LOGGER.info("Skip to download Flink {}_{} as it is already downloaded.", flinkVersion, scalaVersion); + patchFlinkScala(targetFlinkHomeFolder, flinkVersion, scalaVersion); return targetFlinkHomeFolder.getAbsolutePath(); } File flinkTGZ = new File(flinkDownloadFolder, @@ -541,9 +542,52 @@ public static String downloadFlink(String flinkVersion, String scalaVersion) { } catch (Exception e) { throw new RuntimeException("Fail to download jar", e); } + patchFlinkScala(targetFlinkHomeFolder, flinkVersion, scalaVersion); return targetFlinkHomeFolder.getAbsolutePath(); } + /** + * Flink 1.x tgz bundles flink-scala_2.12 which shades scala 2.12.7. + * Scala 2.12.7's JrtClassPath.asURLs() crashes on JDK 17+ (scala/bug#11608, fixed in 2.12.9). + * Strip the shaded classes and copy in Scala 2.12.20 jars. + */ + private static void patchFlinkScala(File flinkHome, String flinkVersion, String scalaVersion) { + if (!"2.12".equals(scalaVersion)) { + return; + } + if (SemanticVersion.of(flinkVersion).equalsOrNewerThan(SemanticVersion.of("2.0.0"))) { + return; + } + File libDir = new File(flinkHome, "lib"); + for (File jar : libDir.listFiles((d, n) -> n.startsWith("flink-scala_") && n.endsWith(".jar"))) { + LOGGER.info("Stripping Scala 2.12.7 classes from {}", jar.getName()); + try { + Process p = new ProcessBuilder("zip", "-d", jar.getAbsolutePath(), "scala/*") + .redirectErrorStream(true).start(); + IOUtils.toString(p.getInputStream(), StandardCharsets.UTF_8); + p.waitFor(); + } catch (Exception e) { + LOGGER.warn("Failed to strip Scala classes from {}", jar.getName(), e); + } + } + String scalaPatchVersion = "2.12.20"; + for (String artifact : new String[]{"scala-library", "scala-compiler", "scala-reflect"}) { + String jarName = artifact + "-" + scalaPatchVersion + ".jar"; + File dest = new File(libDir, jarName); + if (dest.exists()) { + continue; + } + String url = "https://repo1.maven.org/maven2/org/scala-lang/" + artifact + "/" + + scalaPatchVersion + "/" + jarName; + try { + LOGGER.info("Downloading {} to {}", jarName, dest); + download(url, 3, dest); + } catch (Exception e) { + throw new RuntimeException("Fail to download " + jarName, e); + } + } + } + private static void mvFile(String srcPath, String dstPath) throws IOException { Path src = Paths.get(srcPath); Path dst = Paths.get(dstPath);