Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch to hardcoding Smile as the state format #28610

Merged
merged 1 commit into from
Feb 12, 2018
Merged

Switch to hardcoding Smile as the state format #28610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1293,7 +1293,7 @@ public static Settings addHumanReadableSettings(Settings settings) {
/**
* State format for {@link IndexMetaData} to write to and load from disk
*/
public static final MetaDataStateFormat<IndexMetaData> FORMAT = new MetaDataStateFormat<IndexMetaData>(XContentType.SMILE, INDEX_STATE_FILE_PREFIX) {
public static final MetaDataStateFormat<IndexMetaData> FORMAT = new MetaDataStateFormat<IndexMetaData>(INDEX_STATE_FILE_PREFIX) {

@Override
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1205,7 +1205,7 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
/**
* State format for {@link MetaData} to write to and load from disk
*/
public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) {
public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {

@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/env/NodeMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public NodeMetaData build() {
}


public static final MetaDataStateFormat<NodeMetaData> FORMAT = new MetaDataStateFormat<NodeMetaData>(XContentType.SMILE, "node-") {
public static final MetaDataStateFormat<NodeMetaData> FORMAT = new MetaDataStateFormat<NodeMetaData>("node-") {

@Override
protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {
Expand Down
23 changes: 9 additions & 14 deletions server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,36 +64,28 @@
* @param <T> the type of the XContent base data-structure
*/
public abstract class MetaDataStateFormat<T> {
public static final XContentType FORMAT = XContentType.SMILE;
public static final String STATE_DIR_NAME = "_state";
public static final String STATE_FILE_EXTENSION = ".st";

private static final String STATE_FILE_CODEC = "state";
private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 0;
private static final int STATE_FILE_VERSION = 1;
private static final int STATE_FILE_VERSION_ES_2X_AND_BELOW = 0;
private static final int BUFFER_SIZE = 4096;
private final XContentType format;
private final String prefix;
private final Pattern stateFilePattern;


/**
* Creates a new {@link MetaDataStateFormat} instance
* @param format the format of the x-content
*/
protected MetaDataStateFormat(XContentType format, String prefix) {
this.format = format;
protected MetaDataStateFormat(String prefix) {
this.prefix = prefix;
this.stateFilePattern = Pattern.compile(Pattern.quote(prefix) + "(\\d+)(" + MetaDataStateFormat.STATE_FILE_EXTENSION + ")?");

}

/**
* Returns the {@link XContentType} used to serialize xcontent on write.
*/
public XContentType format() {
return format;
}

/**
* Writes the given state to the given directories. The state is written to a
* state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it
Expand Down Expand Up @@ -123,8 +115,8 @@ public final void write(final T state, final Path... locations) throws IOExcepti
try (OutputStreamIndexOutput out =
new OutputStreamIndexOutput(resourceDesc, fileName, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) {
CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
out.writeInt(format.index());
try (XContentBuilder builder = newXContentBuilder(format, new IndexOutputOutputStream(out) {
out.writeInt(FORMAT.index());
try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) {
@Override
public void close() throws IOException {
// this is important since some of the XContentBuilders write bytes on close.
Expand Down Expand Up @@ -190,14 +182,17 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro
final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION,
STATE_FILE_VERSION);
final XContentType xContentType = XContentType.values()[indexInput.readInt()];
if (xContentType != FORMAT) {
throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType);
}
if (fileVersion == STATE_FILE_VERSION_ES_2X_AND_BELOW) {
// format version 0, wrote a version that always came from the content state file and was never used
indexInput.readLong(); // version currently unused
}
long filePointer = indexInput.getFilePointer();
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(namedXContentRegistry,
try (XContentParser parser = XContentFactory.xContent(FORMAT).createParser(namedXContentRegistry,
new InputStreamIndexInput(slice, contentSize))) {
return fromXContent(parser);
}
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public String toString() {
return "primary [" + primary + "], allocation [" + allocationId + "]";
}

public static final MetaDataStateFormat<ShardStateMetaData> FORMAT = new MetaDataStateFormat<ShardStateMetaData>(XContentType.SMILE, SHARD_STATE_FILE_PREFIX) {
public static final MetaDataStateFormat<ShardStateMetaData> FORMAT = new MetaDataStateFormat<ShardStateMetaData>(SHARD_STATE_FILE_PREFIX) {

@Override
protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {
Expand Down
21 changes: 10 additions & 11 deletions server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
* Ensure we can read a pre-generated cluster state.
*/
public void testReadClusterState() throws URISyntaxException, IOException {
final MetaDataStateFormat<MetaData> format = new MetaDataStateFormat<MetaData>(randomFrom(XContentType.values()), "global-") {
final MetaDataStateFormat<MetaData> format = new MetaDataStateFormat<MetaData>("global-") {

@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
Expand Down Expand Up @@ -103,7 +103,7 @@ public void testReadWriteState() throws IOException {
dirs[i] = createTempDir();
}
final long id = addDummyFiles("foo-", dirs);
Format format = new Format(randomFrom(XContentType.values()), "foo-");
Format format = new Format("foo-");
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, dirs);
Expand Down Expand Up @@ -145,7 +145,7 @@ public void testVersionMismatch() throws IOException {
}
final long id = addDummyFiles("foo-", dirs);

Format format = new Format(randomFrom(XContentType.values()), "foo-");
Format format = new Format("foo-");
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, dirs);
Expand All @@ -169,7 +169,7 @@ public void testCorruption() throws IOException {
dirs[i] = createTempDir();
}
final long id = addDummyFiles("foo-", dirs);
Format format = new Format(randomFrom(XContentType.values()), "foo-");
Format format = new Format("foo-");
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, dirs);
Expand Down Expand Up @@ -244,17 +244,16 @@ public void testLoadState() throws IOException {
meta.add(randomMeta());
}
Set<Path> corruptedFiles = new HashSet<>();
MetaDataStateFormat<MetaData> format = metaDataFormat(randomFrom(XContentType.values()));
MetaDataStateFormat<MetaData> format = metaDataFormat();
for (int i = 0; i < dirs.length; i++) {
dirs[i] = createTempDir();
Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
for (int j = 0; j < numLegacy; j++) {
XContentType type = format.format();
if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {
Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
} else {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type,
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(MetaDataStateFormat.FORMAT,
Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
xcontentBuilder.startObject();
MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, ToXContent.EMPTY_PARAMS);
Expand Down Expand Up @@ -309,8 +308,8 @@ public void testLoadState() throws IOException {
}
}

private static MetaDataStateFormat<MetaData> metaDataFormat(XContentType format) {
return new MetaDataStateFormat<MetaData>(format, MetaData.GLOBAL_STATE_FILE_PREFIX) {
private static MetaDataStateFormat<MetaData> metaDataFormat() {
return new MetaDataStateFormat<MetaData>(MetaData.GLOBAL_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
MetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS);
Expand Down Expand Up @@ -347,8 +346,8 @@ private IndexMetaData.Builder indexBuilder(String index) throws IOException {

private class Format extends MetaDataStateFormat<DummyState> {

Format(XContentType format, String prefix) {
super(format, prefix);
Format(String prefix) {
super(prefix);
}

@Override
Expand Down