Update to Elasticsearch 7.14.0

Author: spinscale

README.md

@@ -10,6 +10,7 @@ Note that Elasticsearch has native support for langdetection nowadays using the
 
 | ES    | Command |
 | ----- | ------- |
+| 7.14.0 | `bin/elasticsearch-plugin install https://github.com/spinscale/elasticsearch-ingest-langdetect/releases/download/7.14.0.1/ingest-langdetect-7.14.0.1.zip` |
 | 7.13.4 | `bin/elasticsearch-plugin install https://github.com/spinscale/elasticsearch-ingest-langdetect/releases/download/7.13.4.1/ingest-langdetect-7.13.4.1.zip` |
 | 7.13.3 | `bin/elasticsearch-plugin install https://github.com/spinscale/elasticsearch-ingest-langdetect/releases/download/7.13.3.1/ingest-langdetect-7.13.3.1.zip` |
 | 7.13.2 | `bin/elasticsearch-plugin install https://github.com/spinscale/elasticsearch-ingest-langdetect/releases/download/7.13.2.1/ingest-langdetect-7.13.2.1.zip` |

build.gradle

@@ -1,4 +1,12 @@
 import java.nio.file.Files
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.gradle.api.tasks.Input;
+import org.gradle.process.CommandLineArgumentProvider;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 buildscript {
   dependencies {
@@ -22,31 +30,22 @@ version = "${elasticsearchVersion}.1-SNAPSHOT"
 apply plugin: 'java'
 apply plugin: 'idea'
 apply plugin: 'elasticsearch.esplugin'
-apply plugin: 'elasticsearch.yaml-rest-test'
-
-// license of this project
-licenseFile = rootProject.file('LICENSE.txt')
-// copyright notices
-noticeFile = rootProject.file('NOTICE.txt')
+apply plugin: 'elasticsearch.testclusters'
 
 esplugin {
-  name 'ingest-langdetect'
-  description 'Ingest processor doing language detection for fields'
-  classname 'org.elasticsearch.plugin.ingest.langdetect.IngestLangDetectPlugin'
-  // license of the plugin, may be different than the above license
-  licenseFile rootProject.file('LICENSE.txt')
-  // copyright notices, may be different than the above notice
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'ingest-langdetect'
+  description = 'Ingest processor doing language detection for fields'
+  classname = 'org.elasticsearch.plugin.ingest.langdetect.IngestLangDetectPlugin'
+  licenseFile = rootProject.file('LICENSE.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
-validateElasticPom.enabled = false
-
 // In this section you declare the dependencies for your production and test code
 dependencies {
   implementation 'com.youcruit.com.cybozu.labs:langdetect:1.1.2-20151117'
   implementation 'net.arnx:jsonic:1.3.10'
   // the yaml tests require a log4j2 dependency, otherwise a dependency is thrown on startup
-  yamlRestTestImplementation 'org.apache.logging.log4j:log4j-core:2.11.1'
+  runtimeOnly 'org.apache.logging.log4j:log4j-core:2.11.1'
 }
 
 // ignore javadoc warnings for now
@@ -76,3 +75,97 @@ githubRelease.doFirst {
     assets = [ filename ]
   }
 }
+
+
+// setup yaml rest tests
+testClusters {
+  yamlRestTest
+}
+
+sourceSets {
+  yamlRestTest
+}
+
+configurations {
+  yamlRestTestImplementation.extendsFrom testImplementation
+  yamlRestTestRuntimeOnly.extendsFrom testRuntimeOnly
+  restTestSpecs
+}
+
+tasks.register('copyRestTestSpecs', Copy) {
+  from zipTree(configurations.restTestSpecs.singleFile)
+  into "$buildDir/restResources/restspec"
+}
+
+TaskProvider<Zip> bundle = project.getTasks().withType(Zip.class).named("bundlePlugin");
+
+// Register rest resources with source set
+sourceSets.yamlRestTest.getOutput().dir("$buildDir/restResources/restspec");
+
+tasks.register('yamlRestTest', StandaloneRestIntegTestTask) { testTask ->
+    testTask.dependsOn(bundle, 'copyRestTestSpecs')
+
+    def cluster = testClusters.yamlRestTest
+    cluster.plugin(bundle.flatMap(AbstractArchiveTask::getArchiveFile))
+    testTask.useCluster(testClusters.yamlRestTest)
+
+    testTask.mustRunAfter(project.getTasks().named("test"))
+    testTask.setTestClassesDirs(sourceSets.yamlRestTest.getOutput().getClassesDirs())
+    testTask.setClasspath(sourceSets.yamlRestTest.getRuntimeClasspath())
+
+
+    SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider()
+    nonInputProperties.systemProperty("tests.rest.cluster", "${-> String.join(",", cluster.getAllHttpSocketURI())}")
+    nonInputProperties.systemProperty("tests.cluster", "${-> String.join(",", cluster.getAllTransportPortURI())}")
+    nonInputProperties.systemProperty("tests.clustername", "${-> cluster.getName()}")
+    testTask.getJvmArgumentProviders().add(nonInputProperties)
+    testTask.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString())
+}
+
+// this is a bit of a hack to make sure we run the test tests when releasing...
+check.dependsOn 'yamlRestTest'
+
+dependencies {
+  yamlRestTestImplementation "org.elasticsearch.test:framework:$elasticsearchVersion"
+  restTestSpecs "org.elasticsearch:rest-api-spec:$elasticsearchVersion"
+}
+
+// remove when 7.14.1 is released
+tasks.withType(Test).configureEach { testTask ->
+  testTask.systemProperties 'gradle.dist.lib': "${gradle.gradleHomeDir}/lib",
+        'gradle.worker.jar': "${gradle.gradleUserHomeDir}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar",
+        'tests.gradle': 'true',
+        'tests.task': testTask.path
+}
+
+// This will be available in 7.15 in build tools and not manually declared. 
+public class SystemPropertyCommandLineArgumentProvider implements CommandLineArgumentProvider {
+    private final Map<String, Object> systemProperties = new LinkedHashMap<>();
+
+    public void systemProperty(String key, Supplier<String> value) {
+        systemProperties.put(key, value);
+    }
+
+    public void systemProperty(String key, Object value) {
+        systemProperties.put(key, value);
+    }
+
+    @Override
+    public Iterable<String> asArguments() {
+        return systemProperties.entrySet()
+            .stream()
+            .map(
+                entry -> "-D"
+                    + entry.getKey()
+                    + "="
+                    + (entry.getValue() instanceof Supplier ? ((Supplier) entry.getValue()).get() : entry.getValue())
+            )
+            .collect(Collectors.toList());
+    }
+
+    // Track system property keys as an input so our build cache key will change if we add properties but values are still ignored
+    @Input
+    public Iterable<String> getPropertyNames() {
+        return systemProperties.keySet();
+    }
+}

gradle.properties

@@ -1 +1 @@
-elasticsearchVersion = 7.13.4
+elasticsearchVersion = 7.14.0

src/test/java/org/elasticsearch/plugin/ingest/langdetect/LangDetectProcessorTests.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.util.Collections;
@@ -44,6 +45,11 @@ public static void loadProfiles() throws Exception {
         SecureDetectorFactory.loadProfileFromClassPath(environment);
     }
 
+    @AfterClass
+    public static void done() {
+        System.out.println("DONE");
+    }
+
     public void testThatProcessorWorks() throws Exception {
         Map<String, Object> data = ingestDocument(config("source_field", "language", false),
                 "source_field", "This is hopefully an english text, that will be detected.");