Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add reusable HistogramValue object #49799

Merged
merged 7 commits into from
Dec 4, 2019
Merged

Add reusable HistogramValue object #49799

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
76e0c15
Add a reusable implementation of HistogramValue so we do not create
iverase Dec 3, 2019
990b3ba
checkStyle
iverase Dec 3, 2019
b43a086
Deserialize using ByteArrayDataInput
iverase Dec 3, 2019
dac0fb1
remove unused imports
iverase Dec 3, 2019
876d97f
Use ByteBuffersDataOutput
iverase Dec 4, 2019
85de8a0
Use Double to bits methods instead of NumericUtils
iverase Dec 4, 2019
56c8ca9
rename streamInput to dataInput
iverase Dec 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 58 additions & 49 deletions ...nalytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
Expand All @@ -49,7 +49,6 @@
import org.elasticsearch.search.MultiValueMode;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -202,7 +201,9 @@ public AtomicHistogramFieldData load(LeafReaderContext context) {
public HistogramValues getHistogramValues() throws IOException {
try {
final BinaryDocValues values = DocValues.getBinary(context.reader(), fieldName);
final InternalHistogramValue value = new InternalHistogramValue();
return new HistogramValues() {

@Override
public boolean advanceExact(int doc) throws IOException {
return values.advanceExact(doc);
Expand All @@ -211,7 +212,8 @@ public boolean advanceExact(int doc) throws IOException {
@Override
public HistogramValue histogram() throws IOException {
try {
return getHistogramValue(values.binaryValue());
value.reset(values.binaryValue());
return value;
} catch (IOException e) {
throw new IOException("Cannot load doc value", e);
}
Expand All @@ -220,7 +222,6 @@ public HistogramValue histogram() throws IOException {
} catch (IOException e) {
throw new IOException("Cannot load doc values", e);
}

}

@Override
Expand Down Expand Up @@ -259,44 +260,6 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode,
}
};
}

private HistogramValue getHistogramValue(final BytesRef bytesRef) throws IOException {
final ByteBufferStreamInput streamInput = new ByteBufferStreamInput(
ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length));
return new HistogramValue() {
double value;
int count;
boolean isExhausted;

@Override
public boolean next() throws IOException {
if (streamInput.available() > 0) {
count = streamInput.readVInt();
value = streamInput.readDouble();
return true;
}
isExhausted = true;
return false;
}

@Override
public double value() {
if (isExhausted) {
throw new IllegalArgumentException("histogram already exhausted");
}
return value;
}

@Override
public int count() {
if (isExhausted) {
throw new IllegalArgumentException("histogram already exhausted");
}
return count;
}
};
}

};
}

Expand Down Expand Up @@ -395,21 +358,20 @@ public void parse(ParseContext context) throws IOException {
"[" + COUNTS_FIELD.getPreferredName() +"] but got [" + values.size() + " != " + counts.size() +"]");
}
if (fieldType().hasDocValues()) {
BytesStreamOutput streamOutput = new BytesStreamOutput();
ByteBuffersDataOutput dataOutput = new ByteBuffersDataOutput();
for (int i = 0; i < values.size(); i++) {
int count = counts.get(i);
if (count < 0) {
throw new MapperParsingException("error parsing field ["
+ name() + "], ["+ COUNTS_FIELD + "] elements must be >= 0 but got " + counts.get(i));
} else if (count > 0) {
// we do not add elements with count == 0
streamOutput.writeVInt(count);
streamOutput.writeDouble(values.get(i));
dataOutput.writeVInt(count);
dataOutput.writeLong(Double.doubleToRawLongBits(values.get(i)));
}
}

Field field = new BinaryDocValuesField(simpleName(), streamOutput.bytes().toBytesRef());
streamOutput.close();
BytesRef docValue = new BytesRef(dataOutput.toArrayCopy(), 0, Math.toIntExact(dataOutput.size()));
Field field = new BinaryDocValuesField(simpleName(), docValue);
if (context.doc().getByKey(fieldType().name()) != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() +
"] doesn't not support indexing multiple values for the same field in the same document");
Expand Down Expand Up @@ -439,4 +401,51 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
builder.field(Names.IGNORE_MALFORMED, ignoreMalformed.value());
}
}

/** re-usable {@link HistogramValue} implementation */
private static class InternalHistogramValue extends HistogramValue {
double value;
int count;
boolean isExhausted;
ByteArrayDataInput dataInput;

InternalHistogramValue() {
dataInput = new ByteArrayDataInput();
}

/** reset the value for the histogram */
void reset(BytesRef bytesRef) {
dataInput.reset(bytesRef.bytes, bytesRef.offset, bytesRef.length);
isExhausted = false;
value = 0;
count = 0;
}

@Override
public boolean next() {
if (dataInput.eof() == false) {
count = dataInput.readVInt();
value = Double.longBitsToDouble(dataInput.readLong());
return true;
}
isExhausted = true;
return false;
}

@Override
public double value() {
if (isExhausted) {
throw new IllegalArgumentException("histogram already exhausted");
}
return value;
}

@Override
public int count() {
if (isExhausted) {
throw new IllegalArgumentException("histogram already exhausted");
}
return count;
}
}
}