Backport java 17 build support to 7.17 (#1950)

This commit adds support for building on java 17, while leaving support for building java 11 untouched.

Author: masseyke

.ci/java-versions.properties

@@ -6,5 +6,5 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ESH_BUILD_JAVA=openjdk14
-ESH_RUNTIME_JAVA=java11
+ESH_BUILD_JAVA=openjdk17
+ESH_RUNTIME_JAVA=openjdk17

hive/build.gradle

@@ -58,6 +58,13 @@ dependencies {
     javadocSources(project(":elasticsearch-hadoop-mr"))
 }
 
+tasks.named("test").configure {
+    if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+        jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+        jvmArgs "--add-opens=java.base/java.net=ALL-UNNAMED" // Needed for org.apache.hadoop.hive.common.StringInternUtils
+    }
+}
+
 jar {
     dependsOn(project.configurations.embedded)
     from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {

pig/build.gradle

@@ -47,6 +47,12 @@ dependencies {
     javadocSources(project(":elasticsearch-hadoop-mr"))
 }
 
+tasks.named("test").configure {
+    if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+        jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+    }
+}
+
 jar {
     dependsOn(project.configurations.embedded)
     from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {

spark/core/build.gradle

@@ -26,7 +26,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -144,8 +143,14 @@ configurations.matching{ it.name.contains('CompilerPlugin') == false }.all { Con
     conf.exclude group: "org.mortbay.jetty"
 }
 
-tasks.named("test").configure {
-    jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+if (JavaVersion.current() >= JavaVersion.VERSION_17) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test")) 
+            task.configure {
+                jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+                jvmArgs "--add-opens=java.base/java.nio=ALL-UNNAMED" // Needed for org.apache.spark.SparkConf, which indirectly uses java.nio.DirectByteBuffer
+            }
+    }
 }
 
 // Set minimum compatibility and java home for compiler task

spark/sql-13/build.gradle

@@ -36,7 +36,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -176,6 +175,14 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
 }
 
+if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test"))
+        task.configure {
+            jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+        }}
+}
+
 // Embed the embedded dependencies in the final jar after all configuration is complete
 sparkVariants {
     all { SparkVariantPlugin.SparkVariant variant ->

spark/sql-20/build.gradle

@@ -32,7 +32,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -196,6 +195,14 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
 }
 
+if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test"))
+        task.configure {
+            jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+        }}
+}
+
 // Embed the embedded dependencies in the final jar after all configuration is complete
 sparkVariants {
     all { SparkVariantPlugin.SparkVariant variant ->

spark/sql-30/build.gradle

@@ -30,7 +30,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -179,6 +178,15 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
 }
 
+if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test"))
+        task.configure {
+            jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+            jvmArgs "--add-opens=java.base/java.nio=ALL-UNNAMED" // Needed for org.apache.spark.SparkConf, which indirectly uses java.nio.DirectByteBuffer
+        }}
+}
+
 // Embed the embedded dependencies in the final jar after all configuration is complete
 sparkVariants {
     all { SparkVariantPlugin.SparkVariant variant ->

test/shared/build.gradle

@@ -58,6 +58,12 @@ configurations {
     }
 }
 
+tasks.named("test").configure {
+    if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+        jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+    }
+}
+
 dependencies {
     api(project(':elasticsearch-hadoop-mr'))
     api(project.ext.hadoopClient) {