Add tool elasticsearch-node unsafe-bootstrap

distribution/src/bin/elasticsearch-node

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ES_MAIN_CLASS=org.elasticsearch.cluster.coordination.NodeToolCli \
+  "`dirname "$0"`"/elasticsearch-cli \
+  "$@"

distribution/src/bin/elasticsearch-node.bat

@@ -0,0 +1,12 @@
+@echo off
+
+setlocal enabledelayedexpansion
+setlocal enableextensions
+
+set ES_MAIN_CLASS=org.elasticsearch.cluster.coordination.NodeToolCli
+call "%~dp0elasticsearch-cli.bat" ^
+  %%* ^
+  || exit /b 1
+
+endlocal
+endlocal

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -301,7 +301,7 @@ public void test90SecurityCliPackaging() {
         }
     }
 
-    public void test100RepairIndexCliPackaging() {
+    public void test100ElasticsearchShardCliPackaging() {
         assumeThat(installation, is(notNullValue()));
 
         final Installation.Executables bin = installation.executables();
@@ -318,4 +318,22 @@ public void test100RepairIndexCliPackaging() {
         }
     }
 
+    public void test110ElasticsearchNodeCliPackaging() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Installation.Executables bin = installation.executables();
+        final Shell sh = new Shell();
+
+        Platforms.PlatformAction action = () -> {
+            final Result result = sh.run(bin.elasticsearchNode + " -h");
+            assertThat(result.stdout,
+                    containsString("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes"));
+        };
+
+        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+            Platforms.onLinux(action);
+            Platforms.onWindows(action);
+        }
+    }
+
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java

@@ -186,7 +186,8 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch-env",
             "elasticsearch-keystore",
             "elasticsearch-plugin",
-            "elasticsearch-shard"
+            "elasticsearch-shard",
+            "elasticsearch-node"
         ).forEach(executable -> {
 
             assertThat(es.bin(executable), file(File, owner, owner, p755));

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -102,6 +102,7 @@ public class Executables {
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
         public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
         public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
+        public final Path elasticsearchNode = platformExecutable("elasticsearch-node");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java

@@ -196,7 +196,8 @@ private static void verifyOssInstallation(Installation es, Distribution distribu
             "elasticsearch",
             "elasticsearch-plugin",
             "elasticsearch-keystore",
-            "elasticsearch-shard"
+            "elasticsearch-shard",
+            "elasticsearch-node"
         ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
 
         Stream.of(

qa/vagrant/src/test/resources/packaging/utils/packages.bash

@@ -104,6 +104,7 @@ verify_package_installation() {
     assert_file "$ESHOME/bin/elasticsearch" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
     assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
+    assert_file "$ESHOME/bin/elasticsearch-node" f root root 755
     assert_file "$ESHOME/lib" d root root 755
     assert_file "$ESCONFIG" d root elasticsearch 2750
     assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660

qa/vagrant/src/test/resources/packaging/utils/tar.bash

@@ -95,6 +95,7 @@ verify_archive_installation() {
     assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
     assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
+    assert_file "$ESHOME/bin/elasticsearch-node" f elasticsearch elasticsearch 755
     assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
     assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660
     assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660

server/src/main/java/org/elasticsearch/cli/CommandLoggingConfigurator.java

@@ -27,13 +27,13 @@
  * Holder class for method to configure logging without Elasticsearch configuration files for use in CLI tools that will not read such
  * files.
  */
-final class CommandLoggingConfigurator {
+public final class CommandLoggingConfigurator {
 
     /**
      * Configures logging without Elasticsearch configuration files based on the system property "es.logger.level" only. As such, any
      * logging will be written to the console.
      */
-    static void configureLoggingWithoutConfig() {
+    public static void configureLoggingWithoutConfig() {
         // initialize default for es.logger.level because we will not read the log4j2.properties
         final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name());
         final Settings settings = Settings.builder().put("logger.level", loggerLevel).build();

server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.cli.CommandLoggingConfigurator;
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
+
+// NodeToolCli does not extend LoggingAwareCommand, because LoggingAwareCommand performs logging initialization
+// after LoggingAwareCommand instance is constructed.
+// It's too late for us, because before UnsafeBootstrapMasterCommand is added to the list of subcommands
+// log4j2 initialization will happen, because it has static reference to Logger class.
+// Even if we avoid making a static reference to Logger class, there is no nice way to avoid declaring
+// UNSAFE_BOOTSTRAP, which depends on ClusterService, which in turn has static Logger.
+// TODO execute CommandLoggingConfigurator.configureLoggingWithoutConfig() in the constructor of commands, not in beforeMain
+public class NodeToolCli extends MultiCommand {
+
+    public NodeToolCli() {
+        super("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes", ()->{});
+        CommandLoggingConfigurator.configureLoggingWithoutConfig();
+        subcommands.put("unsafe-bootstrap", new UnsafeBootstrapMasterCommand());
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new NodeToolCli().main(args, Terminal.DEFAULT));
+    }
+
+}

server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java

@@ -0,0 +1,187 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import joptsimple.OptionSet;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.Manifest;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.NodeMetaData;
+import org.elasticsearch.node.Node;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Objects;
+
+public class UnsafeBootstrapMasterCommand extends EnvironmentAwareCommand {
+
+    private static final Logger logger = LogManager.getLogger(UnsafeBootstrapMasterCommand.class);
+    private final NamedXContentRegistry namedXContentRegistry;
+
+    static final String WARNING_MSG =
+            "-----------------------------------------------------------------------\n" +
+            "\n" +
+            "    WARNING: Elasticsearch MUST be stopped before running this tool.\n" +
+            "\n" +
+            "You should run this tool only if you have permanently lost half\n" +
+            "or more of the master-eligible nodes, and you cannot restore the cluster\n" +
+            "from a snapshot. This tool can result in arbitrary data loss and " +
+            "should be the last resort.\n" +
+            "Do you accept this risk?\n";
+    static final String ABORTED_BY_USER_MSG = "aborted by user";
+    static final String NOT_MASTER_NODE_MSG = "unsafe-bootstrap tool can only be run on master eligible node";
+    static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?";
+    static final String NO_NODE_FOLDER_FOUND_MSG = "no node folder is found in data folder(s), node has not been started yet?";
+    static final String NO_NODE_METADATA_FOUND_MSG = "no node meta data is found, node has not been started yet?";
+    static final String NO_MANIFEST_FILE_FOUND_MSG = "no manifest file is found, do you run pre 7.0 Elasticsearch?";
+    static final String GLOBAL_GENERATION_MISSING_MSG = "no metadata is referenced from the manifest file, cluster has never been " +
+            "bootstrapped?";
+    static final String NO_GLOBAL_METADATA_MSG = "failed to find global metadata, metadata corrupted?";
+    static final String EMPTY_LAST_COMMITTED_VOTING_CONFIG_MSG =
+            "last committed voting voting configuration is empty, cluster has never been bootstrapped?";
+    static final String WRITE_METADATA_EXCEPTION_MSG = "exception occurred when writing new metadata to disk";
+    static final String MASTER_NODE_BOOTSTRAPPED_MSG = "Master node was successfully bootstrapped";
+    static final Setting<String> UNSAFE_BOOTSTRAP =
+            ClusterService.USER_DEFINED_META_DATA.getConcreteSetting("cluster.metadata.unsafe-bootstrap");
+
+    UnsafeBootstrapMasterCommand() {
+        super("Forces the successful election of the current node after the permanent loss of the half or more master-eligible nodes");
+        namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        showWarning(terminal);
+        String text = terminal.readText("Confirm [y/N] ");
+        if (text.equalsIgnoreCase("y") == false) {
+            throw new ElasticsearchException(ABORTED_BY_USER_MSG);
+        }
+
+        Settings settings = env.settings();
+        terminal.println(Terminal.Verbosity.VERBOSE, "Checking node.master setting");
+        Boolean master = Node.NODE_MASTER_SETTING.get(settings);
+        if (master == false) {
+            throw new ElasticsearchException(NOT_MASTER_NODE_MSG);
+        }
+        final int nodeOrdinal = 0;
+
+        terminal.println(Terminal.Verbosity.VERBOSE, "Obtaining lock for node");
+
+        try (NodeEnvironment.NodeLock lock = new NodeEnvironment.NodeLock(nodeOrdinal, logger, env, Files::exists)) {
+            processNodePaths(logger, terminal, lock.getNodePaths());
+        } catch (LockObtainFailedException ex) {
+            throw new ElasticsearchException(
+                    FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]");
+        }
+
+        terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG);
+    }
+
+    private void processNodePaths(Logger logger, Terminal terminal, NodeEnvironment.NodePath[] nodePaths) throws IOException {
+        final Path[] dataPaths =
+                Arrays.stream(nodePaths).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new);
+        if (dataPaths.length == 0) {
+            throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG);
+        }
+
+        terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
+        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        if (nodeMetaData == null) {
+            throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
+        }
+
+        String nodeId = nodeMetaData.nodeId();
+        terminal.println(Terminal.Verbosity.VERBOSE, "Current nodeId is " + nodeId);
+        terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
+        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+
+        if (manifest == null) {
+            throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
+        }
+        if (manifest.isGlobalGenerationMissing()) {
+            throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
+        }
+        terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
+        final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
+                dataPaths);
+        if (metaData == null) {
+            throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
+        }
+        final CoordinationMetaData coordinationMetaData = metaData.coordinationMetaData();
+        if (coordinationMetaData == null ||
+                coordinationMetaData.getLastCommittedConfiguration() == null ||
+                coordinationMetaData.getLastCommittedConfiguration().isEmpty()) {
+            throw new ElasticsearchException(EMPTY_LAST_COMMITTED_VOTING_CONFIG_MSG);
+        }
+
+        CoordinationMetaData newCoordinationMetaData = CoordinationMetaData.builder(coordinationMetaData)
+                .clearVotingConfigExclusions()
+                .lastAcceptedConfiguration(new CoordinationMetaData.VotingConfiguration(Collections.singleton(nodeId)))
+                .lastCommittedConfiguration(new CoordinationMetaData.VotingConfiguration(Collections.singleton(nodeId)))
+                .build();
+        terminal.println(Terminal.Verbosity.VERBOSE, "New coordination metadata is constructed " + newCoordinationMetaData);
+        Settings persistentSettings = Settings.builder()
+                .put(metaData.persistentSettings())
+                .put(UNSAFE_BOOTSTRAP.getKey(), true)
+                .build();
+        MetaData newMetaData = MetaData.builder(metaData)
+                .persistentSettings(persistentSettings)
+                .coordinationMetaData(newCoordinationMetaData)
+                .build();
+        writeNewMetaData(terminal, manifest, newMetaData, dataPaths);
+    }
+
+    private void writeNewMetaData(Terminal terminal, Manifest manifest, MetaData newMetaData, Path[] dataPaths) {
+        try {
+            terminal.println(Terminal.Verbosity.VERBOSE, "Writing new global metadata to disk");
+            long newGeneration = MetaData.FORMAT.write(newMetaData, dataPaths);
+            long newCurrentTerm = manifest.getCurrentTerm() + 1;
+            terminal.println(Terminal.Verbosity.VERBOSE, "Incrementing currentTerm. New value is " + newCurrentTerm);
+            Manifest newManifest = new Manifest(newCurrentTerm, manifest.getClusterStateVersion(), newGeneration,
+                    manifest.getIndexGenerations());
+            terminal.println(Terminal.Verbosity.VERBOSE, "Writing new manifest file to disk");
+            Manifest.FORMAT.writeAndCleanup(newManifest, dataPaths);
+            terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up old metadata");
+            MetaData.FORMAT.cleanupOldFiles(newGeneration, dataPaths);
+        } catch (Exception e) {
+            terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up new metadata");
+            MetaData.FORMAT.cleanupOldFiles(manifest.getGlobalGeneration(), dataPaths);
+            throw new ElasticsearchException(WRITE_METADATA_EXCEPTION_MSG, e);
+        }
+    }
+
+    private void showWarning(Terminal terminal) {
+        terminal.println(WARNING_MSG);
+    }
+}