Remove Spark 2.x (#2316)

Author: samxbr

dist/build.gradle

@@ -12,7 +12,7 @@ base {
   archivesName = 'elasticsearch-hadoop'
 }
 
-def sparkVariantIncluded = 'spark20scala211'
+def sparkVariantIncluded = 'spark30scala212'
 
 configurations {
     embedded {
@@ -47,17 +47,17 @@ configurations {
 
 BuildPlugin.disableTransitiveDependencies(project, project.configurations.thirdPartyShaded)
 
-def distProjects = [":elasticsearch-hadoop-mr", ":elasticsearch-hadoop-hive",
-                    ":elasticsearch-spark-20"]
+def distProjects = [":elasticsearch-hadoop-mr", ":elasticsearch-hadoop-hive", ":elasticsearch-spark-30"]
 
 distProjects.each { distProject ->
     def configureDistDependency = { Dependency dependency ->
-        if (distProject == ":elasticsearch-spark-20") {
+        if (distProject == ":elasticsearch-spark-30") {
             dependency.capabilities {
                 requireCapability("org.elasticsearch.spark.sql.variant:$sparkVariantIncluded:$project.version")
             }
         }
     }
+
     dependencies {
         // This is only going to pull in each project's regular jar to create the project-wide uberjar.
         add('embedded', project(distProject), configureDistDependency)
@@ -86,25 +86,25 @@ dependencies {
     }
     implementation("org.apache.hive:hive-exec:$hiveVersion")
     implementation("org.apache.hive:hive-metastore:$hiveVersion")
-    implementation("org.apache.spark:spark-core_${project.ext.scala211MajorVersion}:$spark20Version") {
+    implementation("org.apache.spark:spark-core_${project.ext.scala212MajorVersion}:$spark30Version") {
         exclude group: 'javax.servlet'
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.apache.spark:spark-yarn_${project.ext.scala211MajorVersion}:$spark20Version") {
+    implementation("org.apache.spark:spark-yarn_${project.ext.scala212MajorVersion}:$spark30Version") {
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.apache.spark:spark-sql_${project.ext.scala211MajorVersion}:$spark20Version") {
+    implementation("org.apache.spark:spark-sql_${project.ext.scala212MajorVersion}:$spark30Version") {
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.apache.spark:spark-streaming_${project.ext.scala211MajorVersion}:$spark20Version") {
+    implementation("org.apache.spark:spark-streaming_${project.ext.scala212MajorVersion}:$spark30Version") {
         exclude group: 'org.apache.hadoop'
     }
     implementation("org.scala-lang:scala-library:$scala211Version")
     implementation("org.scala-lang:scala-reflect:$scala211Version")
     implementation(project.ext.hadoopClient)
     implementation("org.apache.hadoop:hadoop-common:${project.ext.hadoopVersion}")
     implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${project.ext.hadoopVersion}")
-    compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala211MajorVersion}:$spark20Version")
+    compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala212MajorVersion}:$spark30Version")
 }
 
 // Configure uber jar

dist/licenses/spark-core_2.11-2.3.0.jar.sha1

@@ -0,0 +1 @@
+6f7105e792cbb650eca44890f0444720bd6a8204

dist/licenses/spark-core_2.12-3.4.3.jar.sha1

@@ -0,0 +1 @@
+a28ead1aa19899654296a6f8a458f8f207f89a73

dist/licenses/spark-sql_2.11-2.3.0.jar.sha1

@@ -0,0 +1 @@
+e4a9f76eff1db09c67f855cdf60cd83bbfad69ab

dist/licenses/spark-sql_2.12-3.4.3.jar.sha1

@@ -0,0 +1 @@
+0c8dd1e45fbc589d2438a889c7fe98f0e9fd77ec

dist/licenses/spark-streaming_2.11-2.3.0.jar.sha1

@@ -8,6 +8,16 @@ For clarity, we always list any breaking changes at the top of the
 //NOTE: The notable-breaking-changes tagged regions are re-used in the
 //Installation and Upgrade Guide
 
+[[breaking-changes-90]]
+=== Breaking Changes in 9.0
+
+This section details the breaking changes when upgrading {eh} from 8.x to 9.0.
+
+[[removals-9.0]]
+==== Removal of Spark 2.x
+
+Support for the Spark 2.x has been removed in {eh} 9.0. Spark 3.x is the new default supported version.
+
 === Deprecations in 8.18
 
 The following functionality has been deprecated in {eh} 8.18 and will be removed