Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 13 additions & 15 deletions .github/workflows/core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
strategy:
fail-fast: false
matrix:
java: [ 11 ]
java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
Expand Down Expand Up @@ -90,7 +90,7 @@ jobs:
strategy:
fail-fast: false
matrix:
java: [ 11 ]
java: [ 11, 17 ]
env:
INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell'
steps:
Expand Down Expand Up @@ -136,7 +136,7 @@ jobs:
fail-fast: false
matrix:
python: [ 3.9 ]
java: [ 11 ]
java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
Expand Down Expand Up @@ -181,7 +181,7 @@ jobs:
strategy:
fail-fast: false
matrix:
java: [ 11 ]
java: [ 11, 17 ]
steps:
# user/password => root/root
- name: Start mysql
Expand Down Expand Up @@ -232,22 +232,24 @@ jobs:
fail-fast: false
matrix:
include:
- python: 3.9
- java: 11
python: 3.9
flink: 119
flink-profile: "1.19"
- python: 3.9
- java: 17
python: 3.9
flink: 120
flink-profile: "1.20"
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Set up JDK 11
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: 11
java-version: ${{ matrix.java }}
- name: Cache local Maven repository
uses: actions/cache@v5
with:
Expand Down Expand Up @@ -285,7 +287,7 @@ jobs:
strategy:
fail-fast: false
matrix:
java: [ 11 ]
java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
Expand Down Expand Up @@ -366,22 +368,18 @@ jobs:
auto-activate: false
use-mamba: true
- name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }}
if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }}
if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }}
if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }}
if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
Expand Down Expand Up @@ -442,7 +440,7 @@ jobs:
strategy:
fail-fast: false
matrix:
java: [ 11 ]
java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
Expand Down Expand Up @@ -472,7 +470,7 @@ jobs:
strategy:
fail-fast: false
matrix:
java: [ 11 ]
java: [ 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
Expand Down
26 changes: 25 additions & 1 deletion bin/common.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,38 @@ if not defined ZEPPELIN_JAVA_OPTS (
set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
)

REM JPMS (Java Platform Module System) args mirrored from pom.xml extraJavaTestArgs.
REM Targets JDK 17+: --add-modules, --enable-native-access, --sun-misc-unsafe-memory-access
REM require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX flags.
set JPMS_JAVA_OPTS=-XX:+IgnoreUnrecognizedVMOptions
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.io=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.net=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.nio=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.security.action=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.tryReflectionSetAccessible=true
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.allocator.type=pooled
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED

if not defined JAVA_OPTS (
set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
) else (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
set JAVA_OPTS=%JAVA_OPTS% %JPMS_JAVA_OPTS%


set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%
set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %JPMS_JAVA_OPTS%

if not defined JAVA_HOME (
set ZEPPELIN_RUNNER=java
Expand Down
26 changes: 24 additions & 2 deletions bin/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -147,15 +147,37 @@ if [[ ( -z "${ZEPPELIN_INTP_MEM}" ) && ( "${ZEPPELIN_INTERPRETER_LAUNCHER}" != "
export ZEPPELIN_INTP_MEM="-Xmx1024m"
fi

JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}"
# JPMS (Java Platform Module System) args mirrored from pom.xml extraJavaTestArgs.
# Targets JDK 17+: --add-modules, --enable-native-access, --sun-misc-unsafe-memory-access
# require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX flags.
JPMS_JAVA_OPTS="-XX:+IgnoreUnrecognizedVMOptions"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.cs=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.security.action=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
JPMS_JAVA_OPTS+=" -Dio.netty.tryReflectionSetAccessible=true"
JPMS_JAVA_OPTS+=" -Dio.netty.allocator.type=pooled"
JPMS_JAVA_OPTS+=" -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE"
JPMS_JAVA_OPTS+=" --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED"
JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM} ${JPMS_JAVA_OPTS}"
if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties"
else
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
fi
export JAVA_OPTS

JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${JPMS_JAVA_OPTS}"
if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties -Dlog4j.configurationFile=file://${ZEPPELIN_CONF_DIR}/log4j2_docker.properties"
elif [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
Expand Down
64 changes: 62 additions & 2 deletions flink/flink-scala-2.12/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
<properties>
<!--library versions-->
<flink.version>${flink1.19.version}</flink.version>
<flink.scala.version>2.12.7</flink.scala.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
<flink.scala.compile.version>${flink.scala.version}</flink.scala.compile.version>
<hive.version>2.3.7</hive.version>
Expand Down Expand Up @@ -1060,7 +1059,7 @@
<!-- set sun.zip.disableMemoryMapping=true because of
https://blogs.oracle.com/poonam/crashes-in-zipgetentry
https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8191484 -->
<argLine>-Xmx5120m -XX:MaxMetaspaceSize=1024m -Dsun.zip.disableMemoryMapping=true</argLine>
<argLine>-Xmx5120m -XX:MaxMetaspaceSize=1024m -Dsun.zip.disableMemoryMapping=true ${extraJavaTestArgs}</argLine>
<!-- <argLine>-Xmx4096m -XX:MaxMetaspaceSize=512m -Dsun.zip.disableMemoryMapping=true -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=6006</argLine>-->

<environmentVariables>
Expand Down Expand Up @@ -1222,6 +1221,38 @@
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- Flink 1.x tgz bundles flink-scala_2.12 which shades scala 2.12.7.
2.12.7's JrtClassPath.asURLs() crashes on JDK 17+ (scala/bug#11608, fixed in 2.12.9).
Strip the shaded classes and drop in 2.12.20 jars. Not needed for Flink 2.0+. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>patch-flink-scala</id>
<phase>initialize</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>
<exec executable="zip" dir="${project.build.directory}/flink-${flink.version}/lib" failonerror="true">
<arg line="-d flink-scala_${flink.scala.binary.version}-${flink.version}.jar scala/*"/>
</exec>
<copy todir="${project.build.directory}/flink-${flink.version}/lib" flatten="true">
<resources>
<file file="${settings.localRepository}/org/scala-lang/scala-library/${scala.2.12.version}/scala-library-${scala.2.12.version}.jar"/>
<file file="${settings.localRepository}/org/scala-lang/scala-compiler/${scala.2.12.version}/scala-compiler-${scala.2.12.version}.jar"/>
<file file="${settings.localRepository}/org/scala-lang/scala-reflect/${scala.2.12.version}/scala-reflect-${scala.2.12.version}.jar"/>
</resources>
</copy>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

<profile>
Expand Down Expand Up @@ -1256,6 +1287,35 @@
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>patch-flink-scala</id>
<phase>initialize</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>
<exec executable="zip" dir="${project.build.directory}/flink-${flink.version}/lib" failonerror="true">
<arg line="-d flink-scala_${flink.scala.binary.version}-${flink.version}.jar scala/*"/>
</exec>
<copy todir="${project.build.directory}/flink-${flink.version}/lib" flatten="true">
<resources>
<file file="${settings.localRepository}/org/scala-lang/scala-library/${scala.2.12.version}/scala-library-${scala.2.12.version}.jar"/>
<file file="${settings.localRepository}/org/scala-lang/scala-compiler/${scala.2.12.version}/scala-compiler-${scala.2.12.version}.jar"/>
<file file="${settings.localRepository}/org/scala-lang/scala-reflect/${scala.2.12.version}/scala-reflect-${scala.2.12.version}.jar"/>
</resources>
</copy>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion flink/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<flink1.19.version>1.19.3</flink1.19.version>
<flink1.20.version>1.20.4</flink1.20.version>

<flink.scala.version>2.12.7</flink.scala.version>
<flink.scala.version>2.12.20</flink.scala.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
</properties>
</project>
25 changes: 24 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,29 @@
<testcontainers.version>1.21.4</testcontainers.version>

<MaxMetaspace>512m</MaxMetaspace>
<extraJavaTestArgs>
-XX:+IgnoreUnrecognizedVMOptions
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-Dio.netty.tryReflectionSetAccessible=true
-Dio.netty.allocator.type=pooled
-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
--sun-misc-unsafe-memory-access=allow
--enable-native-access=ALL-UNNAMED
-XX:+EnableDynamicAgentLoading
</extraJavaTestArgs>

<!-- to be able to exclude some tests using command line -->
<tests.to.exclude/>
Expand Down Expand Up @@ -658,7 +681,7 @@
<configuration combine.children="append">
<failIfNoTests>false</failIfNoTests>
<failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs}</argLine>
<environmentVariables>
<IS_ZEPPELIN_TEST>true</IS_ZEPPELIN_TEST>
</environmentVariables>
Expand Down
32 changes: 0 additions & 32 deletions spark/interpreter/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,6 @@
<!-- settings -->
<pyspark.test.exclude>**/PySparkInterpreterMatplotlibTest.java</pyspark.test.exclude>
<pyspark.test.include>**/*Test.*</pyspark.test.include>

<extraJavaTestArgs></extraJavaTestArgs>
</properties>

<dependencies>
Expand Down Expand Up @@ -453,36 +451,6 @@
</build>

<profiles>

<profile>
<id>java-17</id>
<activation>
<jdk>[17,)</jdk>
</activation>
<properties>
<extraJavaTestArgs>
-XX:+IgnoreUnrecognizedVMOptions
--add-modules=jdk.incubator.vector
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-Djdk.reflect.useDirectMethodHandle=false
-Dio.netty.tryReflectionSetAccessible=true
</extraJavaTestArgs>
</properties>
</profile>

<!-- profile spark-scala-x only affect the unit test in spark/interpreter module -->

<profile>
Expand Down
2 changes: 1 addition & 1 deletion zeppelin-integration/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration combine.children="append">
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8 ${extraJavaTestArgs}</argLine>
<excludes>
<exclude>${tests.to.exclude}</exclude>
</excludes>
Expand Down
2 changes: 1 addition & 1 deletion zeppelin-interpreter-integration/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<argLine>-Xmx3072m</argLine>
<argLine>-Xmx3072m ${extraJavaTestArgs}</argLine>
<environmentVariables>
<ZEPPELIN_HOME>${basedir}/../</ZEPPELIN_HOME>
</environmentVariables>
Expand Down
Loading
Loading