Skip to content

Commit

Permalink
Node repurpose tool
Browse files Browse the repository at this point in the history
When a node is repurposed to master/no-data or no-master/no-data, v7.x
will not start (see elastic#37748 and elastic#37347). The `elasticsearch repurpose`
tool can fix this by cleaning up the problematic data.
  • Loading branch information
henningandersen committed Feb 26, 2019
1 parent 406633e commit 73b17b4
Show file tree
Hide file tree
Showing 8 changed files with 735 additions and 40 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.elasticsearch.cluster.coordination;

import joptsimple.OptionSet;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.metadata.Manifest;
import org.elasticsearch.cluster.metadata.MetaData;
Expand All @@ -45,24 +44,18 @@ public DetachClusterCommand() {
super("Detaches this node from its cluster, allowing it to unsafely join a new cluster");
}

@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
super.execute(terminal, options, env);

processNodePathsWithLock(terminal, options, env);

terminal.println(NODE_DETACHED_MSG);
}

@Override
protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException {
protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
final Tuple<Manifest, MetaData> manifestMetaDataTuple = loadMetaData(terminal, dataPaths);
final Manifest manifest = manifestMetaDataTuple.v1();
final MetaData metaData = manifestMetaDataTuple.v2();

confirm(terminal, CONFIRMATION_MSG);

writeNewMetaData(terminal, manifest, updateCurrentTerm(), metaData, updateMetaData(metaData), dataPaths);

terminal.println(NODE_DETACHED_MSG);
}

// package-private for tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,14 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
"\n" +
" WARNING: Elasticsearch MUST be stopped before running this tool." +
"\n";
static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?";
protected static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?";
static final String NO_NODE_FOLDER_FOUND_MSG = "no node folder is found in data folder(s), node has not been started yet?";
static final String NO_MANIFEST_FILE_FOUND_MSG = "no manifest file is found, do you run pre 7.0 Elasticsearch?";
static final String GLOBAL_GENERATION_MISSING_MSG = "no metadata is referenced from the manifest file, cluster has never been " +
"bootstrapped?";
protected static final String GLOBAL_GENERATION_MISSING_MSG =
"no metadata is referenced from the manifest file, cluster has never been bootstrapped?";
static final String NO_GLOBAL_METADATA_MSG = "failed to find global metadata, metadata corrupted?";
static final String WRITE_METADATA_EXCEPTION_MSG = "exception occurred when writing new metadata to disk";
static final String ABORTED_BY_USER_MSG = "aborted by user";
protected static final String ABORTED_BY_USER_MSG = "aborted by user";
final OptionSpec<Integer> nodeOrdinalOption;

public ElasticsearchNodeCommand(String description) {
Expand All @@ -78,7 +78,7 @@ protected void processNodePathsWithLock(Terminal terminal, OptionSet options, En
if (dataPaths.length == 0) {
throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG);
}
processNodePaths(terminal, dataPaths);
processNodePaths(terminal, dataPaths, env);
} catch (LockObtainFailedException ex) {
throw new ElasticsearchException(
FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]");
Expand Down Expand Up @@ -114,11 +114,31 @@ protected void confirm(Terminal terminal, String msg) {
}

@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
protected final void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
terminal.println(STOP_WARNING_MSG);
if (validateBeforeLock(terminal, env)) {
processNodePathsWithLock(terminal, options, env);
}
}

/**
* Validate that the command can run before taking any locks.
* @param terminal the terminal to print to
* @param env the env to validate.
* @return true to continue, false to stop (must print message in validate).
*/
protected boolean validateBeforeLock(Terminal terminal, Environment env) {
return true;
}

protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException;

/**
* Process the paths. Locks for the paths is held during this method invocation.
* @param terminal the terminal to use for messages
* @param dataPaths the paths of the node to process
* @param env the env of the node to process
*/
protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException;


protected void writeNewMetaData(Terminal terminal, Manifest oldManifest, long newCurrentTerm,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.cli.CommandLoggingConfigurator;
import org.elasticsearch.cli.MultiCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.env.NodeRepurposeCommand;

// NodeToolCli does not extend LoggingAwareCommand, because LoggingAwareCommand performs logging initialization
// after LoggingAwareCommand instance is constructed.
Expand All @@ -36,6 +37,7 @@ public NodeToolCli() {
CommandLoggingConfigurator.configureLoggingWithoutConfig();
subcommands.put("unsafe-bootstrap", new UnsafeBootstrapMasterCommand());
subcommands.put("detach-cluster", new DetachClusterCommand());
subcommands.put("repurpose", new NodeRepurposeCommand());
}

public static void main(String[] args) throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.elasticsearch.cluster.coordination;

import joptsimple.OptionSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
Expand Down Expand Up @@ -70,22 +69,18 @@ public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand {
}

@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
super.execute(terminal, options, env);

protected boolean validateBeforeLock(Terminal terminal, Environment env) {
Settings settings = env.settings();
terminal.println(Terminal.Verbosity.VERBOSE, "Checking node.master setting");
Boolean master = Node.NODE_MASTER_SETTING.get(settings);
if (master == false) {
throw new ElasticsearchException(NOT_MASTER_NODE_MSG);
}

processNodePathsWithLock(terminal, options, env);

terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG);
return true;
}

protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException {
protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
if (nodeMetaData == null) {
Expand Down Expand Up @@ -128,5 +123,7 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOEx
.build();

writeNewMetaData(terminal, manifest, manifest.getCurrentTerm(), metaData, newMetaData, dataPaths);

terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG);
}
}
50 changes: 35 additions & 15 deletions server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,8 @@

package org.elasticsearch.env;

import java.io.UncheckedIOException;
import java.util.Iterator;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SegmentInfos;
Expand All @@ -34,22 +30,22 @@
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.NativeFSLockFactory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
Expand All @@ -63,6 +59,7 @@

import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileStore;
Expand All @@ -74,6 +71,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand All @@ -84,6 +82,8 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static java.util.Collections.unmodifiableSet;

Expand Down Expand Up @@ -1041,28 +1041,48 @@ private static void ensureAtomicMoveSupported(final NodePath[] nodePaths) throws
}

private void ensureNoShardData(final NodePath[] nodePaths) throws IOException {
List<Path> shardDataPaths = collectIndexSubPaths(nodePaths, this::isShardPath);
List<Path> shardDataPaths = collectShardDataPaths(nodePaths);
if (shardDataPaths.isEmpty() == false) {
throw new IllegalStateException("Node is started with "
+ Node.NODE_DATA_SETTING.getKey()
+ "=false, but has shard data: "
+ shardDataPaths);
+ shardDataPaths
+ ". Use 'elasticsearch-node repurpose' tool to clean up"
);
}
}

private void ensureNoIndexMetaData(final NodePath[] nodePaths) throws IOException {
List<Path> indexMetaDataPaths = collectIndexSubPaths(nodePaths, this::isIndexMetaDataPath);
List<Path> indexMetaDataPaths = collectIndexMetaDataPaths(nodePaths);
if (indexMetaDataPaths.isEmpty() == false) {
throw new IllegalStateException("Node is started with "
+ Node.NODE_DATA_SETTING.getKey()
+ "=false and "
+ Node.NODE_MASTER_SETTING.getKey()
+ "=false, but has index metadata: "
+ indexMetaDataPaths);
+ indexMetaDataPaths
+ ". Use 'elasticsearch-node repurpose' tool to clean up"
);
}
}

private List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> subPathPredicate) throws IOException {
/**
* Collect the paths containing shard data in the indicated node paths. The returned paths will point to the shard data folder.
*/
static List<Path> collectShardDataPaths(NodePath[] nodePaths) throws IOException {
return collectIndexSubPaths(nodePaths, NodeEnvironment::isShardPath);
}


/**
* Collect the paths containing index meta data in the indicated node paths. The returned paths will point to the
* {@link MetaDataStateFormat#STATE_DIR_NAME} folder
*/
static List<Path> collectIndexMetaDataPaths(NodePath[] nodePaths) throws IOException {
return collectIndexSubPaths(nodePaths, NodeEnvironment::isIndexMetaDataPath);
}

private static List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> subPathPredicate) throws IOException {
List<Path> indexSubPaths = new ArrayList<>();
for (NodePath nodePath : nodePaths) {
Path indicesPath = nodePath.indicesPath;
Expand All @@ -1084,12 +1104,12 @@ private List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> su
return indexSubPaths;
}

private boolean isShardPath(Path path) {
private static boolean isShardPath(Path path) {
return Files.isDirectory(path)
&& path.getFileName().toString().chars().allMatch(Character::isDigit);
}

private boolean isIndexMetaDataPath(Path path) {
private static boolean isIndexMetaDataPath(Path path) {
return Files.isDirectory(path)
&& path.getFileName().toString().equals(MetaDataStateFormat.STATE_DIR_NAME);
}
Expand Down
Loading

0 comments on commit 73b17b4

Please sign in to comment.