Integrate better with lucene test framework and mockfilesystems

TESTING.asciidoc

@@ -260,7 +260,7 @@ The REST tests are run automatically when executing the maven test command. To r
 REST tests use the following command:
 
 ---------------------------------------------------------------------------
-mvn test -Dtests.class=org.elasticsearch.test.rest.ElasticsearchRestTests
+mvn test -Dtests.filter="@Rest"
 ---------------------------------------------------------------------------
 
 `ElasticsearchRestTests` is the executable test class that runs all the

dev-tools/forbidden/all-signatures.txt

@@ -33,3 +33,6 @@ java.nio.file.Path#toFile()
 @defaultMessage Don't use deprecated lucene apis
 org.apache.lucene.index.DocsEnum
 org.apache.lucene.index.DocsAndPositionsEnum
+
+java.nio.file.Paths @ Use PathUtils.get instead.
+java.nio.file.FileSystems#getDefault() @ use PathUtils.getDefault instead.

dev-tools/forbidden/test-signatures.txt

@@ -16,3 +16,5 @@
 
 com.carrotsearch.randomizedtesting.RandomizedTest#globalTempDir() @ Use newTempDirPath() instead
 com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds
+
+org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead

pom.xml

@@ -32,15 +32,17 @@
 
     <properties>
         <lucene.version>5.2.0</lucene.version>
-        <lucene.maven.version>5.2.0-snapshot-1674183</lucene.maven.version>
+        <lucene.maven.version>5.2.0-snapshot-1674576</lucene.maven.version>
+        <testframework.version>2.1.14</testframework.version>
         <tests.jvms>auto</tests.jvms>
         <tests.shuffle>true</tests.shuffle>
         <tests.output>onerror</tests.output>
         <tests.client.ratio></tests.client.ratio>
         <tests.bwc.path>${project.basedir}/backwards</tests.bwc.path>
         <tests.locale>random</tests.locale>
         <tests.timezone>random</tests.timezone>
-        <es.logger.level>INFO</es.logger.level>
+        <tests.slow>false</tests.slow>
+        <es.logger.level>ERROR</es.logger.level>
         <tests.heap.size>512m</tests.heap.size>
         <tests.heapdump.path>${basedir}/logs/</tests.heapdump.path>
         <tests.topn>5</tests.topn>
@@ -66,7 +68,7 @@
         <repository>
             <id>lucene-snapshots</id>
             <name>Lucene Snapshots</name>
-            <url>https://download.elastic.co/lucenesnapshots/1674183</url>
+            <url>https://download.elastic.co/lucenesnapshots/1674576</url>
         </repository>
     </repositories>
 
@@ -80,7 +82,7 @@
         <dependency>
             <groupId>com.carrotsearch.randomizedtesting</groupId>
             <artifactId>randomizedtesting-runner</artifactId>
-            <version>2.1.13</version>
+            <version>${testframework.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -501,7 +503,7 @@
             <plugin>
                 <groupId>com.carrotsearch.randomizedtesting</groupId>
                 <artifactId>junit4-maven-plugin</artifactId>
-                <version>2.1.13</version>
+                <version>${testframework.version}</version>
                 <executions>
                     <execution>
                         <id>tests</id>
@@ -510,9 +512,10 @@
                             <goal>junit4</goal>
                         </goals>
                         <configuration>
-                            <heartbeat>20</heartbeat>
+                            <heartbeat>10</heartbeat>
                             <jvmOutputAction>pipe,warn</jvmOutputAction>
                             <leaveTemporary>true</leaveTemporary>
+                            <ifNoTests>fail</ifNoTests>
                             <listeners>
                                 <report-ant-xml mavenExtensions="true"
                                                 dir="${project.build.directory}/surefire-reports"/>
@@ -525,7 +528,19 @@
                                         showStatusFailure="true"
                                         showStatusIgnored="true"
                                         showSuiteSummary="true"
-                                        timestamps="false"/>
+                                        timestamps="false">
+                                  <filtertrace>
+                                     <!-- custom filters: we carefully only omit test infra noise here -->
+                                     <containsstring contains=".SlaveMain." />
+                                     <containsregex pattern="^(\s+at )(org\.junit\.)" />
+                                     <!-- also includes anonymous classes inside these two: -->
+                                     <containsregex pattern="^(\s+at )(com\.carrotsearch\.randomizedtesting.RandomizedRunner)" />
+                                     <containsregex pattern="^(\s+at )(com\.carrotsearch\.randomizedtesting.ThreadLeakControl)" />
+                                     <containsregex pattern="^(\s+at )(com\.carrotsearch\.randomizedtesting.rules\.)" />
+                                     <containsregex pattern="^(\s+at )(org\.apache\.lucene.util\.TestRule)" />
+                                     <containsregex pattern="^(\s+at )(org\.apache\.lucene.util\.AbstractBeforeAfterRule)" />
+                                  </filtertrace>
+                                </report-text>
                                 <report-execution-times historyLength="20" file="${basedir}/${execution.hint.file}"/>
                             </listeners>
                             <assertions>
@@ -561,7 +576,8 @@
                             <sysouts>${tests.verbose}</sysouts>
                             <seed>${tests.seed}</seed>
                             <haltOnFailure>${tests.failfast}</haltOnFailure>
-                            <uniqueSuiteNames>false</uniqueSuiteNames>
+                            <!-- enforce unique suite names, or reporting stuff can be screwed up -->
+                            <uniqueSuiteNames>true</uniqueSuiteNames>
                             <systemProperties>
                                 <!-- we use './temp' since this is per JVM and tests are forbidden from writing to CWD -->
                                 <java.io.tmpdir>./temp</java.io.tmpdir>
@@ -570,7 +586,6 @@
                                 <tests.bwc.path>${tests.bwc.path}</tests.bwc.path>
                                 <tests.bwc.version>${tests.bwc.version}</tests.bwc.version>
                                 <tests.jvm.argline>${tests.jvm.argline}</tests.jvm.argline>
-                                <tests.processors>${tests.processors}</tests.processors>
                                 <tests.appendseed>${tests.appendseed}</tests.appendseed>
                                 <tests.iters>${tests.iters}</tests.iters>
                                 <tests.maxfailures>${tests.maxfailures}</tests.maxfailures>
@@ -1626,6 +1641,7 @@
                     <version>2.9</version>
                     <configuration>
                         <buildOutputDirectory>eclipse-build</buildOutputDirectory>
+                        <downloadSources>true</downloadSources>
                     </configuration>
                 </plugin>
             </plugins>

rest-api-spec/test/README.asciidoc

@@ -65,7 +65,7 @@ skipped, and the reason why the tests are skipped.  For instance:
 ....
     "Parent":
      - skip:
-          version:     "0 - 0.90.2"
+          version:     "0.20.1 - 0.90.2"
           reason:      Delete ignores the parent param
 
      - do:
@@ -75,14 +75,17 @@ skipped, and the reason why the tests are skipped.  For instance:
 All tests in the file following the skip statement should be skipped if:
 `min <= current <= max`.
 
-The `version` range should always have an upper bound. Versions should
-either have each version part compared numerically, or should be converted
-to a string with sufficient digits to allow string comparison, eg
-
-    0.90.2 -> 000-090-002
+The `version` range can leave either bound empty, which means "open ended".
+For instance:
+....
+    "Parent":
+     - skip:
+          version:     "1.0.0.Beta1 - "
+          reason:      Delete ignores the parent param
 
-Snapshot versions and versions of the form `1.0.0.Beta1` can be treated
-as the rounded down version, eg `1.0.0`.
+     - do:
+       ... test definitions ...
+....
 
 The skip section can also be used to list new features that need to be
 supported in order to run a test. This way the up-to-date runners will

rest-api-spec/test/cluster.put_settings/10_basic.yaml

@@ -1,7 +1,7 @@
 ---
 setup:
   - skip:
-      version: 0 - 999
+      version:  " - "
       reason:  leaves transient metadata behind, need to fix it
 ---
 "Test put settings":

rest-api-spec/test/indices.get/10_basic.yaml

@@ -166,7 +166,7 @@ setup:
 "Should return test_index_3 if expand_wildcards=closed":
 
   - skip:
-      version:     "0 - 2.0.0"
+      version:     " - 2.0.0"
       reason:      Requires fix for issue 7258
 
   - do:

rest-api-spec/test/indices.get_aliases/10_basic.yaml

@@ -202,7 +202,7 @@ setup:
 "Getting alias on an non-existent index should return 404":
 
   - skip:
-        version: 1 - 999
+        version: "1.0.0.Beta1 - "
         reason:  not implemented yet
   - do:
       catch: missing

rest-api-spec/test/indices.put_settings/all_path_options.yaml

@@ -81,7 +81,7 @@ setup:
 ---
 "put settings in list of indices":
   - skip:
-      version: 1 - 999
+      version: " - "
       reason:  list of indices not implemented yet
   - do:
       indices.put_settings:

rest-api-spec/test/update/85_fields_meta.yaml

@@ -2,7 +2,7 @@
 "Metadata Fields":
 
   - skip:
-      version:  "0 - 999"
+      version:  " - "
       reason:   "Update doesn't return metadata fields, waiting for #3259"
 
   - do:

src/main/java/org/elasticsearch/Version.java

@@ -485,7 +485,7 @@ public static Version fromString(String version) {
         }
         String[] parts = version.split("\\.");
         if (parts.length < 3 || parts.length > 4) {
-            throw new IllegalArgumentException("the version needs to contain major, minor and revision, and optionally the build");
+            throw new IllegalArgumentException("the version needs to contain major, minor and revision, and optionally the build: " + version);
         }
 
         try {

src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.jna.Natives;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -153,7 +155,7 @@ public static void main(String[] args) {
 
         if (pidFile != null) {
             try {
-                PidFile.create(Paths.get(pidFile), true);
+                PidFile.create(PathUtils.get(pidFile), true);
             } catch (Exception e) {
                 String errorMessage = buildErrorMessage("pid", e);
                 sysError(errorMessage, true);

src/main/java/org/elasticsearch/common/io/FileSystemUtils.java

@@ -208,7 +208,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
                 } else if (suffix != null) {
                     if (!isSameFile(file, path)) {
                         // If it already exists we try to copy this new version appending suffix to its name
-                        path = Paths.get(path.toString().concat(suffix));
+                        path = path.resolveSibling(path.getFileName().toString().concat(suffix));
                         // We just move the file to new dir but with a new name (appended with suffix)
                         Files.move(file, path, StandardCopyOption.REPLACE_EXISTING);
                     }
@@ -258,6 +258,8 @@ public static void move(Path source, Path destination) throws IOException {
             Files.walkFileTree(source, new TreeCopier(source, destination, true));
         }
     }
+
+    // TODO: note that this will fail if source and target are on different NIO.2 filesystems.
 
     static class TreeCopier extends SimpleFileVisitor<Path> {
         private final Path source;

src/main/java/org/elasticsearch/common/io/PathUtils.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io;
+
+import org.elasticsearch.common.SuppressForbidden;
+
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/** 
+ * Utilities for creating a Path from names,
+ * or accessing the default FileSystem.
+ * <p>
+ * This class allows the default filesystem to
+ * be changed during tests.
+ */
+@SuppressForbidden(reason = "accesses the default filesystem by design")
+public final class PathUtils {
+    /** no instantiation */
+    private PathUtils() {}
+
+    /** the actual JDK default */
+    static final FileSystem ACTUAL_DEFAULT = FileSystems.getDefault();
+
+    /** can be changed by tests (via reflection) */
+    private static volatile FileSystem DEFAULT = ACTUAL_DEFAULT;
+
+    /** 
+     * Returns a {@code Path} from name components.
+     * <p>
+     * This works just like {@code Paths.get()}.
+     * Remember: just like {@code Paths.get()} this is NOT A STRING CONCATENATION
+     * UTILITY FUNCTION.
+     * <p>
+     * Remember: this should almost never be used. Usually resolve
+     * a path against an existing one!
+     */
+    public static Path get(String first, String... more) {
+        return DEFAULT.getPath(first, more);
+    }
+
+    /** 
+     * Returns a {@code Path} from a URI
+     * <p>
+     * This works just like {@code Paths.get()}.
+     * <p>
+     * Remember: this should almost never be used. Usually resolve
+     * a path against an existing one!
+     */
+    public static Path get(URI uri) {
+        if (uri.getScheme().equalsIgnoreCase("file")) {
+            return DEFAULT.provider().getPath(uri);
+        } else {
+            return Paths.get(uri);
+        }
+    }
+
+    /**
+     * Returns the default FileSystem.
+     */
+    public static FileSystem getDefaultFileSystem() {
+        return DEFAULT;
+    }
+}

src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java

@@ -36,6 +36,9 @@ public class EsExecutors {
      */
     public static final String PROCESSORS = "processors";
 
+    /** Useful for testing */
+    public static final String DEFAULT_SYSPROP = "es.processors.override";
+
     /**
      * Returns the number of processors available but at most <tt>32</tt>.
      */
@@ -44,7 +47,11 @@ public static int boundedNumberOfProcessors(Settings settings) {
          * ie. >= 48 create too many threads and run into OOM see #3478
          * We just use an 32 core upper-bound here to not stress the system
          * too much with too many created threads */
-        return settings.getAsInt(PROCESSORS, Math.min(32, Runtime.getRuntime().availableProcessors()));
+        int defaultValue = Math.min(32, Runtime.getRuntime().availableProcessors());
+        try {
+            defaultValue = Integer.parseInt(System.getProperty(DEFAULT_SYSPROP));
+        } catch (Throwable ignored) {}
+        return settings.getAsInt(PROCESSORS, defaultValue);
     }
 
     public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing(ThreadFactory threadFactory) {