diff --git a/benchmarks/README.md b/benchmarks/README.md index 5be14bab0ecbc..1e89c3f356f85 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -53,9 +53,9 @@ To get realistic results, you should exercise care when running benchmarks. Here `performance` CPU governor. * Vary the problem input size with `@Param`. * Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses: - * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews + * Add `-prof gc` to the options to check whether the garbage collector runs during a microbenchmarks and skews your results. If so, try to force a GC between runs (`-gc true`) but watch out for the caveats. - * Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots. + * Add `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots. * Have your benchmarks peer-reviewed. ### Don't diff --git a/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java new file mode 100644 index 0000000000000..66de336e7dee8 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import org.elasticsearch.common.time.DateFormatter; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.time.ZoneId; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +@Fork(2) +@Warmup(iterations = 10) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +public class RoundingBenchmark { + private static final DateFormatter FORMATTER = DateFormatter.forPattern("date_optional_time"); + + @Param({ + "2000-01-01 to 2020-01-01", // A super long range + "2000-10-01 to 2000-11-01", // A whole month which is pretty believable + "2000-10-29 to 2000-10-30", // A date right around daylight savings time. + "2000-06-01 to 2000-06-02" // A date fully in one time zone. Should be much faster than above. + }) + public String range; + + @Param({ "java time", "es" }) + public String rounder; + + @Param({ "UTC", "America/New_York" }) + public String zone; + + @Param({ "MONTH_OF_YEAR", "HOUR_OF_DAY" }) + public String timeUnit; + + @Param({ "1", "10000", "1000000", "100000000" }) + public int count; + + private long min; + private long max; + private long[] dates; + private Supplier rounderBuilder; + + @Setup + public void buildDates() { + String[] r = range.split(" to "); + min = FORMATTER.parseMillis(r[0]); + max = FORMATTER.parseMillis(r[1]); + dates = new long[count]; + long date = min; + long diff = (max - min) / dates.length; + for (int i = 0; i < dates.length; i++) { + if (date >= max) { + throw new IllegalStateException("made a bad date [" + date + "]"); + } + dates[i] = date; + date += diff; + } + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.valueOf(timeUnit)).timeZone(ZoneId.of(zone)).build(); + switch (rounder) { + case "java time": + rounderBuilder = rounding::prepareJavaTime; + break; + case "es": + rounderBuilder = () -> rounding.prepare(min, max); + break; + default: + throw new IllegalArgumentException("Expectd rounder to be [java time] or [es]"); + } + } + + @Benchmark + public void round(Blackhole bh) { + Rounding.Prepared rounder = rounderBuilder.get(); + for (int i = 0; i < dates.length; i++) { + bh.consume(rounder.round(dates[i])); + } + } + + @Benchmark + public void nextRoundingValue(Blackhole bh) { + Rounding.Prepared rounder = rounderBuilder.get(); + for (int i = 0; i < dates.length; i++) { + bh.consume(rounder.nextRoundingValue(dates[i])); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/LocalTimeOffset.java b/server/src/main/java/org/elasticsearch/common/LocalTimeOffset.java new file mode 100644 index 0000000000000..4053bfe509a1c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/LocalTimeOffset.java @@ -0,0 +1,642 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneOffsetTransitionRule; +import java.time.zone.ZoneRules; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; + +/** + * Converts utc into local time and back again. + *

+ * "Local time" is defined by some time zone, specifically and {@link ZoneId}. + * At any point in time a particular time zone is at some offset from from + * utc. So converting from utc is as simple as adding the offset. + *

+ * Getting from local time back to utc is harder. Most local times happen once. + * But some local times happen twice. And some don't happen at all. Take, for + * example, the time in my house. Most days I don't touch my clocks and I'm a + * constant offset from UTC. But once in the fall at 2am I roll my clock back. + * So at 5am utc my clocks say 1am. Then at 6am utc my clocks say 1am AGAIN. + * I do similarly terrifying things again in the spring when I skip my clocks + * straight from 1:59am to 3am. + *

+ * So there are two methods to convert from local time back to utc, + * {@link #localToUtc(long, Strategy)} and {@link #localToUtcInThisOffset(long)}. + */ +public abstract class LocalTimeOffset { + /** + * Lookup offsets for a provided zone. This can fail if + * there are many transitions and the provided lookup would be very large. + * + * @return a {@linkplain Lookup} or {@code null} if none could be built + */ + public static Lookup lookup(ZoneId zone, long minUtcMillis, long maxUtcMillis) { + if (minUtcMillis > maxUtcMillis) { + throw new IllegalArgumentException("[" + minUtcMillis + "] must be <= [" + maxUtcMillis + "]"); + } + ZoneRules rules = zone.getRules(); + { + LocalTimeOffset fixed = checkForFixedZone(zone, rules); + if (fixed != null) { + return new FixedLookup(zone, fixed); + } + } + List transitions = collectTransitions(zone, rules, minUtcMillis, maxUtcMillis); + if (transitions == null) { + // The range is too large for us to pre-build all the offsets + return null; + } + if (transitions.size() < 3) { + /* + * Its actually quite common that there are *very* few transitions. + * This case where there are only two transitions covers an entire + * year of data! In any case, it is slightly faster to do the + * "simpler" thing and compare the start times instead of perform + * a binary search when there are so few offsets to look at. + */ + return new LinkedListLookup(zone, minUtcMillis, maxUtcMillis, transitions); + } + return new TransitionArrayLookup(zone, minUtcMillis, maxUtcMillis, transitions); + } + + /** + * Lookup offsets without any known min or max time. This will generally + * fail if the provided zone isn't fixed. + * + * @return a lookup function of {@code null} if none could be built + */ + public static LocalTimeOffset lookupFixedOffset(ZoneId zone) { + return checkForFixedZone(zone, zone.getRules()); + } + + private final long millis; + + private LocalTimeOffset(long millis) { + this.millis = millis; + } + + /** + * Convert a time in utc into a the local time at this offset. + */ + public final long utcToLocalTime(long utcMillis) { + return utcMillis + millis; + } + + /** + * Convert a time in local millis to utc millis using this offset. + *

+ * Important: Callers will rarely want to force + * using this offset and are instead instead interested in picking an appropriate + * offset for some local time that they have rounded down. In that case use + * {@link #localToUtc(long, Strategy)}. + */ + public final long localToUtcInThisOffset(long localMillis) { + return localMillis - millis; + } + + /** + * Convert a local time that occurs during this offset or a previous + * offset to utc, providing a strategy for how to resolve "funny" cases. + * You can use this if you've converted from utc to local, rounded down, + * and then want to convert back to utc and you need fine control over + * how to handle the "funny" edges. + *

+ * This will not help you if you must convert a local time that you've + * rounded up. For that you are on your own. May God + * have mercy on your soul. + */ + public abstract long localToUtc(long localMillis, Strategy strat); + public interface Strategy { + /** + * Handle a local time that never actually happened because a "gap" + * jumped over it. This happens in many time zones when folks wind + * their clocks forwards in the spring. + * + * @return the time in utc representing the local time + */ + long inGap(long localMillis, Gap gap); + /** + * Handle a local time that happened before the start of a gap. + * + * @return the time in utc representing the local time + */ + long beforeGap(long localMillis, Gap gap); + /** + * Handle a local time that happened twice because an "overlap" + * jumped behind it. This happens in many time zones when folks wind + * their clocks back in the fall. + * + * @return the time in utc representing the local time + */ + long inOverlap(long localMillis, Overlap overlap); + /** + * Handle a local time that happened before the start of an overlap. + * + * @return the time in utc representing the local time + */ + long beforeOverlap(long localMillis, Overlap overlap); + } + + /** + * Does this offset contain the provided time? + */ + protected abstract boolean containsUtcMillis(long utcMillis); + + /** + * Find the offset containing the provided time, first checking this + * offset, then its previous offset, the than one's previous offset, etc. + */ + protected abstract LocalTimeOffset offsetContaining(long utcMillis); + + @Override + public String toString() { + return toString(millis); + } + protected abstract String toString(long millis); + + /** + * How to get instances of {@link LocalTimeOffset}. + */ + public abstract static class Lookup { + /** + * Lookup the offset at the provided millis in utc. + */ + public abstract LocalTimeOffset lookup(long utcMillis); + + /** + * If the offset for a range is constant then return it, otherwise + * return {@code null}. + */ + public abstract LocalTimeOffset fixedInRange(long minUtcMillis, long maxUtcMillis); + + /** + * The number of offsets in the lookup. Package private for testing. + */ + abstract int size(); + } + + private static class NoPrevious extends LocalTimeOffset { + NoPrevious(long millis) { + super(millis); + } + + @Override + public long localToUtc(long localMillis, Strategy strat) { + return localToUtcInThisOffset(localMillis); + } + + @Override + protected boolean containsUtcMillis(long utcMillis) { + return true; + } + + @Override + protected LocalTimeOffset offsetContaining(long utcMillis) { + /* + * Since there isn't a previous offset this offset *must* contain + * the provided time. + */ + return this; + } + + @Override + protected String toString(long millis) { + return Long.toString(millis); + } + } + + public abstract static class Transition extends LocalTimeOffset { + private final LocalTimeOffset previous; + private final long startUtcMillis; + + private Transition(long millis, LocalTimeOffset previous, long startUtcMillis) { + super(millis); + this.previous = previous; + this.startUtcMillis = startUtcMillis; + } + + /** + * The offset before the this one. + */ + public LocalTimeOffset previous() { + return previous; + } + + @Override + protected final boolean containsUtcMillis(long utcMillis) { + return utcMillis >= startUtcMillis; + } + + @Override + protected final LocalTimeOffset offsetContaining(long utcMillis) { + if (containsUtcMillis(utcMillis)) { + return this; + } + return previous.offsetContaining(utcMillis); + } + + /** + * The time that this offset started in milliseconds since epoch. + */ + public long startUtcMillis() { + return startUtcMillis; + } + } + + public static class Gap extends Transition { + private final long firstMissingLocalTime; + private final long firstLocalTimeAfterGap; + + private Gap(long millis, LocalTimeOffset previous, long startUtcMillis, long firstMissingLocalTime, long firstLocalTimeAfterGap) { + super(millis, previous, startUtcMillis); + this.firstMissingLocalTime = firstMissingLocalTime; + this.firstLocalTimeAfterGap = firstLocalTimeAfterGap; + assert firstMissingLocalTime < firstLocalTimeAfterGap; + } + + @Override + public long localToUtc(long localMillis, Strategy strat) { + if (localMillis >= firstLocalTimeAfterGap) { + return localToUtcInThisOffset(localMillis); + } + if (localMillis >= firstMissingLocalTime) { + return strat.inGap(localMillis, this); + } + return strat.beforeGap(localMillis, this); + } + + /** + * The first time that is missing from the local time because of this gap. + */ + public long firstMissingLocalTime() { + return firstMissingLocalTime; + } + + @Override + protected String toString(long millis) { + return "Gap of " + millis + "@" + Instant.ofEpochMilli(startUtcMillis()); + } + } + + public static class Overlap extends Transition { + private final long firstOverlappingLocalTime; + private final long firstNonOverlappingLocalTime; + + private Overlap(long millis, LocalTimeOffset previous, long startUtcMillis, + long firstOverlappingLocalTime, long firstNonOverlappingLocalTime) { + super(millis, previous, startUtcMillis); + this.firstOverlappingLocalTime = firstOverlappingLocalTime; + this.firstNonOverlappingLocalTime = firstNonOverlappingLocalTime; + assert firstOverlappingLocalTime < firstNonOverlappingLocalTime; + } + + @Override + public long localToUtc(long localMillis, Strategy strat) { + if (localMillis >= firstNonOverlappingLocalTime) { + return localToUtcInThisOffset(localMillis); + } + if (localMillis >= firstOverlappingLocalTime) { + return strat.inOverlap(localMillis, this); + } + return strat.beforeOverlap(localMillis, this); + } + + /** + * The first local time after the overlap stops. + */ + public long firstNonOverlappingLocalTime() { + return firstNonOverlappingLocalTime; + } + + /** + * The first local time to be appear twice. + */ + public long firstOverlappingLocalTime() { + return firstOverlappingLocalTime; + } + + @Override + protected String toString(long millis) { + return "Overlap of " + millis + "@" + Instant.ofEpochMilli(startUtcMillis()); + } + } + + private static class FixedLookup extends Lookup { + private final ZoneId zone; + private final LocalTimeOffset fixed; + + private FixedLookup(ZoneId zone, LocalTimeOffset fixed) { + this.zone = zone; + this.fixed = fixed; + } + + @Override + public LocalTimeOffset lookup(long utcMillis) { + return fixed; + } + + @Override + public LocalTimeOffset fixedInRange(long minUtcMillis, long maxUtcMillis) { + return fixed; + } + + @Override + int size() { + return 1; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "FixedLookup[for %s at %s]", zone, fixed); + } + } + + /** + * Looks up transitions by checking whether the date is after the start + * of each transition. Simple so fast for small numbers of transitions. + */ + private static class LinkedListLookup extends AbstractManyTransitionsLookup { + private final LocalTimeOffset lastOffset; + private final int size; + + LinkedListLookup(ZoneId zone, long minUtcMillis, long maxUtcMillis, List transitions) { + super(zone, minUtcMillis, maxUtcMillis); + int size = 1; + LocalTimeOffset last = buildNoPrevious(transitions.get(0)); + for (ZoneOffsetTransition t : transitions) { + last = buildTransition(t, last); + size++; + } + this.lastOffset = last; + this.size = size; + } + + @Override + public LocalTimeOffset innerLookup(long utcMillis) { + return lastOffset.offsetContaining(utcMillis); + } + + @Override + int size() { + return size; + } + } + + /** + * Builds an array that can be {@link Arrays#binarySearch(long[], long)}ed + * for the daylight savings time transitions. + */ + private static class TransitionArrayLookup extends AbstractManyTransitionsLookup { + private final LocalTimeOffset[] offsets; + private final long[] transitionOutUtcMillis; + + private TransitionArrayLookup(ZoneId zone, long minUtcMillis, long maxUtcMillis, List transitions) { + super(zone, minUtcMillis, maxUtcMillis); + this.offsets = new LocalTimeOffset[transitions.size() + 1]; + this.transitionOutUtcMillis = new long[transitions.size()]; + this.offsets[0] = buildNoPrevious(transitions.get(0)); + int i = 0; + for (ZoneOffsetTransition t : transitions) { + Transition transition = buildTransition(t, this.offsets[i]); + transitionOutUtcMillis[i] = transition.startUtcMillis(); + i++; + this.offsets[i] = transition; + } + } + + @Override + protected LocalTimeOffset innerLookup(long utcMillis) { + int index = Arrays.binarySearch(transitionOutUtcMillis, utcMillis); + if (index < 0) { + /* + * We're mostly not going to find the exact offset. Instead we'll + * end up at the "insertion point" for the utcMillis. We have no + * plans to insert utcMillis in the array, but the offset that + * contains utcMillis happens to be "insertion point" - 1. + */ + index = -index - 1; + } else { + index++; + } + assert index < offsets.length : "binarySearch did something weird"; + return offsets[index]; + } + + @Override + int size() { + return offsets.length; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "TransitionArrayLookup[for %s between %s and %s]", + zone, Instant.ofEpochMilli(minUtcMillis), Instant.ofEpochMilli(maxUtcMillis)); + } + } + + private abstract static class AbstractManyTransitionsLookup extends Lookup { + protected final ZoneId zone; + protected final long minUtcMillis; + protected final long maxUtcMillis; + + AbstractManyTransitionsLookup(ZoneId zone, long minUtcMillis, long maxUtcMillis) { + this.zone = zone; + this.minUtcMillis = minUtcMillis; + this.maxUtcMillis = maxUtcMillis; + } + + @Override + public final LocalTimeOffset lookup(long utcMillis) { + assert utcMillis >= minUtcMillis; + assert utcMillis <= maxUtcMillis; + return innerLookup(utcMillis); + } + + protected abstract LocalTimeOffset innerLookup(long utcMillis); + + @Override + public final LocalTimeOffset fixedInRange(long minUtcMillis, long maxUtcMillis) { + LocalTimeOffset offset = lookup(maxUtcMillis); + return offset.containsUtcMillis(minUtcMillis) ? offset : null; + } + + protected static NoPrevious buildNoPrevious(ZoneOffsetTransition transition) { + return new NoPrevious(transition.getOffsetBefore().getTotalSeconds() * 1000); + } + + protected static Transition buildTransition(ZoneOffsetTransition transition, LocalTimeOffset previous) { + long utcStart = transition.toEpochSecond() * 1000; + long offsetBeforeMillis = transition.getOffsetBefore().getTotalSeconds() * 1000; + long offsetAfterMillis = transition.getOffsetAfter().getTotalSeconds() * 1000; + if (transition.isGap()) { + long firstMissingLocalTime = utcStart + offsetBeforeMillis; + long firstLocalTimeAfterGap = utcStart + offsetAfterMillis; + return new Gap(offsetAfterMillis, previous, utcStart, firstMissingLocalTime, firstLocalTimeAfterGap); + } + long firstOverlappingLocalTime = utcStart + offsetAfterMillis; + long firstNonOverlappingLocalTime = utcStart + offsetBeforeMillis; + return new Overlap(offsetAfterMillis, previous, utcStart, firstOverlappingLocalTime, firstNonOverlappingLocalTime); + } + } + + private static LocalTimeOffset checkForFixedZone(ZoneId zone, ZoneRules rules) { + if (false == rules.isFixedOffset()) { + return null; + } + LocalTimeOffset fixedTransition = new NoPrevious(rules.getOffset(Instant.EPOCH).getTotalSeconds() * 1000); + return fixedTransition; + } + + /** + * The maximum number of {@link ZoneOffsetTransition} to collect before + * giving up because the date range will be "too big". I picked this number + * fairly arbitrarily with the following goals: + *

    + *
  1. Don't let {@code lookup(Long.MIN_VALUE, Long.MAX_VALUE)} consume all + * the memory in the JVM. + *
  2. It should be much larger than the number of offsets I'm bound to + * collect. + *
+ * {@code 5_000} collects about 2_500 years worth offsets which feels like + * quite a few! + */ + private static final int MAX_TRANSITIONS = 5000; + + /** + * Collect transitions from the provided rules for the provided date range + * into a list we can reason about. If we'd collect more than + * {@link #MAX_TRANSITIONS} rules we'll abort, returning {@code null} + * signaling that {@link LocalTimeOffset} is probably not the implementation + * to use in this case. + *

+ * {@link ZoneRules} gives us access to the local time transition database + * with two method: {@link ZoneRules#getTransitions()} for "fully defined" + * transitions and {@link ZoneRules#getTransitionRules()}. This first one + * is a list of transitions and when the they happened. To get the full + * picture of transitions you pick up from where that one leaves off using + * the rules, which are basically factories that you give the year in local + * time to build a transition for that year. + *

+ * This method collects all of the {@link ZoneRules#getTransitions()} that + * are relevant for the date range and, if our range extends past the last + * transition, calls + * {@link #buildTransitionsFromRules(List, ZoneId, ZoneRules, long, long)} + * to build the remaining transitions to fully describe the range. + */ + private static List collectTransitions(ZoneId zone, ZoneRules rules, long minUtcMillis, long maxUtcMillis) { + long minSecond = minUtcMillis / 1000; + long maxSecond = maxUtcMillis / 1000; + List transitions = new ArrayList<>(); + ZoneOffsetTransition t = null; + Iterator itr = rules.getTransitions().iterator(); + // Skip all transitions that are before our start time + while (itr.hasNext() && (t = itr.next()).toEpochSecond() < minSecond) {} + if (false == itr.hasNext()) { + if (minSecond < t.toEpochSecond() && t.toEpochSecond() < maxSecond) { + transitions.add(t); + } + transitions = buildTransitionsFromRules(transitions, zone, rules, minSecond, maxSecond); + if (transitions != null && transitions.isEmpty()) { + /* + * If there aren't any rules and we haven't accumulated + * any transitions then we grab the last one we saw so we + * have some knowledge of the offset. + */ + transitions.add(t); + } + return transitions; + } + transitions.add(t); + while (itr.hasNext()) { + t = itr.next(); + if (t.toEpochSecond() > maxSecond) { + return transitions; + } + transitions.add(t); + if (transitions.size() > MAX_TRANSITIONS) { + return null; + } + } + return buildTransitionsFromRules(transitions, zone, rules, t.toEpochSecond() + 1, maxSecond); + } + + /** + * Build transitions for every year in our range from the rules + * stored in {@link ZoneRules#getTransitionRules()}. + */ + private static List buildTransitionsFromRules(List transitions, + ZoneId zone, ZoneRules rules, long minSecond, long maxSecond) { + List transitionRules = rules.getTransitionRules(); + if (transitionRules.isEmpty()) { + /* + * Zones like Asia/Kathmandu don't have any rules so we don't + * need to do any of this. + */ + return transitions; + } + int minYear = LocalDate.ofInstant(Instant.ofEpochSecond(minSecond), zone).getYear(); + int maxYear = LocalDate.ofInstant(Instant.ofEpochSecond(maxSecond), zone).getYear(); + + /* + * Record only the rules from the current year that are greater + * than the minSecond so we don't go back in time when coming from + * a fixed transition. + */ + ZoneOffsetTransition lastTransitionFromMinYear = null; + for (ZoneOffsetTransitionRule rule : transitionRules) { + lastTransitionFromMinYear = rule.createTransition(minYear); + if (lastTransitionFromMinYear.toEpochSecond() < minSecond) { + continue; + } + transitions.add(lastTransitionFromMinYear); + if (transitions.size() > MAX_TRANSITIONS) { + return null; + } + } + if (minYear == maxYear) { + if (transitions.isEmpty()) { + // Make sure we have *some* transition to work with. + transitions.add(lastTransitionFromMinYear); + } + return transitions; + } + + // Now build transitions for all of the remaining years. + minYear++; + if (transitions.size() + (maxYear - minYear) * transitionRules.size() > MAX_TRANSITIONS) { + return null; + } + for (int year = minYear; year <= maxYear; year++) { + for (ZoneOffsetTransitionRule rule : transitionRules) { + transitions.add(rule.createTransition(year)); + } + } + return transitions; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/Rounding.java b/server/src/main/java/org/elasticsearch/common/Rounding.java index 32beb11b79257..d0eb5f9218940 100644 --- a/server/src/main/java/org/elasticsearch/common/Rounding.java +++ b/server/src/main/java/org/elasticsearch/common/Rounding.java @@ -20,6 +20,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.common.LocalTimeOffset.Gap; +import org.elasticsearch.common.LocalTimeOffset.Overlap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -44,60 +46,105 @@ import java.time.zone.ZoneRules; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; /** - * A strategy for rounding date/time based values. - * + * A strategy for rounding milliseconds since epoch. + *

* There are two implementations for rounding. - * The first one requires a date time unit and rounds to the supplied date time unit (i.e. quarter of year, day of month) - * The second one allows you to specify an interval to round to + * The first one requires a date time unit and rounds to the supplied date time unit (i.e. quarter of year, day of month). + * The second one allows you to specify an interval to round to. + *

+ * See this + * blog for some background reading. Its super interesting and the links are + * a comedy gold mine. If you like time zones. Or hate them. */ public abstract class Rounding implements Writeable { - public enum DateTimeUnit { WEEK_OF_WEEKYEAR((byte) 1, IsoFields.WEEK_OF_WEEK_BASED_YEAR) { + private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(7); + long roundFloor(long utcMillis) { return DateUtils.roundWeekOfWeekYear(utcMillis); } + + @Override + long extraLocalOffsetLookup() { + return extraLocalOffsetLookup; + } }, YEAR_OF_CENTURY((byte) 2, ChronoField.YEAR_OF_ERA) { + private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(366); + long roundFloor(long utcMillis) { return DateUtils.roundYear(utcMillis); } + + long extraLocalOffsetLookup() { + return extraLocalOffsetLookup; + } }, QUARTER_OF_YEAR((byte) 3, IsoFields.QUARTER_OF_YEAR) { + private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(92); + long roundFloor(long utcMillis) { return DateUtils.roundQuarterOfYear(utcMillis); } + + long extraLocalOffsetLookup() { + return extraLocalOffsetLookup; + } }, MONTH_OF_YEAR((byte) 4, ChronoField.MONTH_OF_YEAR) { + private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(31); + long roundFloor(long utcMillis) { return DateUtils.roundMonthOfYear(utcMillis); } + + long extraLocalOffsetLookup() { + return extraLocalOffsetLookup; + } }, DAY_OF_MONTH((byte) 5, ChronoField.DAY_OF_MONTH) { final long unitMillis = ChronoField.DAY_OF_MONTH.getBaseUnit().getDuration().toMillis(); long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, unitMillis); } + + long extraLocalOffsetLookup() { + return unitMillis; + } }, HOUR_OF_DAY((byte) 6, ChronoField.HOUR_OF_DAY) { final long unitMillis = ChronoField.HOUR_OF_DAY.getBaseUnit().getDuration().toMillis(); long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, unitMillis); } + + long extraLocalOffsetLookup() { + return unitMillis; + } }, MINUTES_OF_HOUR((byte) 7, ChronoField.MINUTE_OF_HOUR) { final long unitMillis = ChronoField.MINUTE_OF_HOUR.getBaseUnit().getDuration().toMillis(); long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, unitMillis); } + + long extraLocalOffsetLookup() { + return unitMillis; + } }, SECOND_OF_MINUTE((byte) 8, ChronoField.SECOND_OF_MINUTE) { final long unitMillis = ChronoField.SECOND_OF_MINUTE.getBaseUnit().getDuration().toMillis(); long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, unitMillis); } + + long extraLocalOffsetLookup() { + return unitMillis; + } }; private final byte id; @@ -117,6 +164,14 @@ long roundFloor(long utcMillis) { */ abstract long roundFloor(long utcMillis); + /** + * When looking up {@link LocalTimeOffset} go this many milliseconds + * in the past from the minimum millis since epoch that we plan to + * look up so that we can see transitions that we might have rounded + * down beyond. + */ + abstract long extraLocalOffsetLookup(); + public byte getId() { return id; } @@ -150,19 +205,59 @@ public void writeTo(StreamOutput out) throws IOException { public abstract byte id(); + /** + * A strategy for rounding milliseconds since epoch. + */ + public interface Prepared { + /** + * Rounds the given value. + */ + long round(long utcMillis); + /** + * Given the rounded value (which was potentially generated by + * {@link #round(long)}, returns the next rounding value. For + * example, with interval based rounding, if the interval is + * 3, {@code nextRoundValue(6) = 9}. + */ + long nextRoundingValue(long utcMillis); + } + /** + * Prepare to round many times. + */ + public abstract Prepared prepare(long minUtcMillis, long maxUtcMillis); + + /** + * Prepare to round many dates over an unknown range. Prefer + * {@link #prepare(long, long)} if you can find the range because + * it'll be much more efficient. + */ + public abstract Prepared prepareForUnknown(); + + /** + * Prepare rounding using java time classes. Package private for testing. + */ + abstract Prepared prepareJavaTime(); + /** * Rounds the given value. + *

+ * Prefer {@link #prepare(long, long)} if rounding many values. */ - public abstract long round(long value); + public final long round(long utcMillis) { + return prepare(utcMillis, utcMillis).round(utcMillis); + } /** - * Given the rounded value (which was potentially generated by {@link #round(long)}, returns the next rounding value. For example, with - * interval based rounding, if the interval is 3, {@code nextRoundValue(6) = 9 }. - * - * @param value The current rounding value - * @return The next rounding value + * Given the rounded value (which was potentially generated by + * {@link #round(long)}, returns the next rounding value. For + * example, with interval based rounding, if the interval is + * 3, {@code nextRoundValue(6) = 9}. + *

+ * Prefer {@link #prepare(long, long)} if rounding many values. */ - public abstract long nextRoundingValue(long value); + public final long nextRoundingValue(long utcMillis) { + return prepare(utcMillis, utcMillis).nextRoundingValue(utcMillis); + } /** * How "offset" this rounding is from the traditional "start" of the period. @@ -245,23 +340,16 @@ public Rounding build() { } static class TimeUnitRounding extends Rounding { - static final byte ID = 1; - /** Since, there is no offset of -1 ms, it is safe to use -1 for non-fixed timezones */ - static final long TZ_OFFSET_NON_FIXED = -1; private final DateTimeUnit unit; private final ZoneId timeZone; private final boolean unitRoundsToMidnight; - /** For fixed offset time zones, this is the offset in milliseconds, otherwise TZ_OFFSET_NON_FIXED */ - private final long fixedOffsetMillis; TimeUnitRounding(DateTimeUnit unit, ZoneId timeZone) { this.unit = unit; this.timeZone = timeZone; this.unitRoundsToMidnight = this.unit.field.getBaseUnit().getDuration().toMillis() > 3600000L; - this.fixedOffsetMillis = timeZone.getRules().isFixedOffset() ? - timeZone.getRules().getOffset(Instant.EPOCH).getTotalSeconds() * 1000 : TZ_OFFSET_NON_FIXED; } TimeUnitRounding(StreamInput in) throws IOException { @@ -314,123 +402,59 @@ private LocalDateTime truncateLocalDateTime(LocalDateTime localDateTime) { } @Override - public long round(long utcMillis) { - // This works as long as the tz offset doesn't change. It is worth getting this case out of the way first, - // as the calculations for fixing things near to offset changes are a little expensive and unnecessary - // in the common case of working with fixed offset timezones (such as UTC). - if (fixedOffsetMillis != TZ_OFFSET_NON_FIXED) { - long localMillis = utcMillis + fixedOffsetMillis; - return unit.roundFloor(localMillis) - fixedOffsetMillis; + public Prepared prepare(long minUtcMillis, long maxUtcMillis) { + long minLookup = minUtcMillis - unit.extraLocalOffsetLookup(); + long maxLookup = maxUtcMillis; + + long unitMillis = 0; + if (false == unitRoundsToMidnight) { + /* + * Units that round to midnight can round down from two + * units worth of millis in the future to find the + * nextRoundingValue. + */ + unitMillis = unit.field.getBaseUnit().getDuration().toMillis(); + maxLookup += 2 * unitMillis; + } + LocalTimeOffset.Lookup lookup = LocalTimeOffset.lookup(timeZone, minLookup, maxLookup); + if (lookup == null) { + // Range too long, just use java.time + return prepareJavaTime(); } - Instant instant = Instant.ofEpochMilli(utcMillis); - if (unitRoundsToMidnight) { - final LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, timeZone); - final LocalDateTime localMidnight = truncateLocalDateTime(localDateTime); - return firstTimeOnDay(localMidnight); - } else { - final ZoneRules rules = timeZone.getRules(); - while (true) { - final Instant truncatedTime = truncateAsLocalTime(instant, rules); - final ZoneOffsetTransition previousTransition = rules.previousTransition(instant); - - if (previousTransition == null) { - // truncateAsLocalTime cannot have failed if there were no previous transitions - return truncatedTime.toEpochMilli(); - } - - Instant previousTransitionInstant = previousTransition.getInstant(); - if (truncatedTime != null && previousTransitionInstant.compareTo(truncatedTime) < 1) { - return truncatedTime.toEpochMilli(); - } - - // There was a transition in between the input time and the truncated time. Return to the transition time and - // round that down instead. - instant = previousTransitionInstant.minusNanos(1_000_000); + LocalTimeOffset fixedOffset = lookup.fixedInRange(minLookup, maxLookup); + if (fixedOffset != null) { + // The time zone is effectively fixed + if (unitRoundsToMidnight) { + return new FixedToMidnightRounding(fixedOffset); } + return new FixedNotToMidnightRounding(fixedOffset, unitMillis); } - } - private long firstTimeOnDay(LocalDateTime localMidnight) { - assert localMidnight.toLocalTime().equals(LocalTime.of(0, 0, 0)) : "firstTimeOnDay should only be called at midnight"; - assert unitRoundsToMidnight : "firstTimeOnDay should only be called if unitRoundsToMidnight"; - - // Now work out what localMidnight actually means - final List currentOffsets = timeZone.getRules().getValidOffsets(localMidnight); - if (currentOffsets.isEmpty() == false) { - // There is at least one midnight on this day, so choose the first - final ZoneOffset firstOffset = currentOffsets.get(0); - final OffsetDateTime offsetMidnight = localMidnight.atOffset(firstOffset); - return offsetMidnight.toInstant().toEpochMilli(); - } else { - // There were no midnights on this day, so we must have entered the day via an offset transition. - // Use the time of the transition as it is the earliest time on the right day. - ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().getTransition(localMidnight); - return zoneOffsetTransition.getInstant().toEpochMilli(); + if (unitRoundsToMidnight) { + return new ToMidnightRounding(lookup); } + return new NotToMidnightRounding(lookup, unitMillis); } - private Instant truncateAsLocalTime(Instant instant, final ZoneRules rules) { - assert unitRoundsToMidnight == false : "truncateAsLocalTime should not be called if unitRoundsToMidnight"; - - LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, timeZone); - final LocalDateTime truncatedLocalDateTime = truncateLocalDateTime(localDateTime); - final List currentOffsets = rules.getValidOffsets(truncatedLocalDateTime); - - if (currentOffsets.isEmpty() == false) { - // at least one possibilities - choose the latest one that's still no later than the input time - for (int offsetIndex = currentOffsets.size() - 1; offsetIndex >= 0; offsetIndex--) { - final Instant result = truncatedLocalDateTime.atOffset(currentOffsets.get(offsetIndex)).toInstant(); - if (result.isAfter(instant) == false) { - return result; - } + @Override + public Prepared prepareForUnknown() { + LocalTimeOffset offset = LocalTimeOffset.lookupFixedOffset(timeZone); + if (offset != null) { + if (unitRoundsToMidnight) { + return new FixedToMidnightRounding(offset); } - - assert false : "rounded time not found for " + instant + " with " + this; - return null; - } else { - // The chosen local time didn't happen. This means we were given a time in an hour (or a minute) whose start - // is missing due to an offset transition, so the time cannot be truncated. - return null; - } - } - - private LocalDateTime nextRelevantMidnight(LocalDateTime localMidnight) { - assert localMidnight.toLocalTime().equals(LocalTime.MIDNIGHT) : "nextRelevantMidnight should only be called at midnight"; - assert unitRoundsToMidnight : "firstTimeOnDay should only be called if unitRoundsToMidnight"; - - switch (unit) { - case DAY_OF_MONTH: - return localMidnight.plus(1, ChronoUnit.DAYS); - case WEEK_OF_WEEKYEAR: - return localMidnight.plus(7, ChronoUnit.DAYS); - case MONTH_OF_YEAR: - return localMidnight.plus(1, ChronoUnit.MONTHS); - case QUARTER_OF_YEAR: - return localMidnight.plus(3, ChronoUnit.MONTHS); - case YEAR_OF_CENTURY: - return localMidnight.plus(1, ChronoUnit.YEARS); - default: - throw new IllegalArgumentException("Unknown round-to-midnight unit: " + unit); + return new FixedNotToMidnightRounding(offset, unit.field.getBaseUnit().getDuration().toMillis()); } + return prepareJavaTime(); } @Override - public long nextRoundingValue(long utcMillis) { + Prepared prepareJavaTime() { if (unitRoundsToMidnight) { - final LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(utcMillis), timeZone); - final LocalDateTime earlierLocalMidnight = truncateLocalDateTime(localDateTime); - final LocalDateTime localMidnight = nextRelevantMidnight(earlierLocalMidnight); - return firstTimeOnDay(localMidnight); - } else { - final long unitSize = unit.field.getBaseUnit().getDuration().toMillis(); - final long roundedAfterOneIncrement = round(utcMillis + unitSize); - if (utcMillis < roundedAfterOneIncrement) { - return roundedAfterOneIncrement; - } else { - return round(utcMillis + 2 * unitSize); - } + return new JavaTimeToMidnightRounding(); } + return new JavaTimeNotToMidnightRounding(unit.field.getBaseUnit().getDuration().toMillis()); } @Override @@ -464,6 +488,253 @@ public boolean equals(Object obj) { public String toString() { return "Rounding[" + unit + " in " + timeZone + "]"; } + + private class FixedToMidnightRounding implements Prepared { + private final LocalTimeOffset offset; + + FixedToMidnightRounding(LocalTimeOffset offset) { + this.offset = offset; + } + + @Override + public long round(long utcMillis) { + return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis))); + } + + @Override + public long nextRoundingValue(long utcMillis) { + // TODO this is used in date range's collect so we should optimize it too + return new JavaTimeToMidnightRounding().nextRoundingValue(utcMillis); + } + } + + private class FixedNotToMidnightRounding implements Prepared { + private final LocalTimeOffset offset; + private final long unitMillis; + + FixedNotToMidnightRounding(LocalTimeOffset offset, long unitMillis) { + this.offset = offset; + this.unitMillis = unitMillis; + } + + @Override + public long round(long utcMillis) { + return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis))); + } + + @Override + public final long nextRoundingValue(long utcMillis) { + return round(utcMillis + unitMillis); + } + } + + private class ToMidnightRounding implements Prepared, LocalTimeOffset.Strategy { + private final LocalTimeOffset.Lookup lookup; + + ToMidnightRounding(LocalTimeOffset.Lookup lookup) { + this.lookup = lookup; + } + + @Override + public long round(long utcMillis) { + LocalTimeOffset offset = lookup.lookup(utcMillis); + return offset.localToUtc(unit.roundFloor(offset.utcToLocalTime(utcMillis)), this); + } + + @Override + public long nextRoundingValue(long utcMillis) { + // TODO this is actually used date range's collect so we should optimize it + return new JavaTimeToMidnightRounding().nextRoundingValue(utcMillis); + } + + @Override + public long inGap(long localMillis, Gap gap) { + return gap.startUtcMillis(); + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + return gap.previous().localToUtc(localMillis, this); + }; + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + return overlap.previous().localToUtc(localMillis, this); + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + return overlap.previous().localToUtc(localMillis, this); + }; + } + + private class NotToMidnightRounding extends AbstractNotToMidnightRounding implements LocalTimeOffset.Strategy { + private final LocalTimeOffset.Lookup lookup; + + NotToMidnightRounding(LocalTimeOffset.Lookup lookup, long unitMillis) { + super(unitMillis); + this.lookup = lookup; + } + + @Override + public long round(long utcMillis) { + LocalTimeOffset offset = lookup.lookup(utcMillis); + long roundedLocalMillis = unit.roundFloor(offset.utcToLocalTime(utcMillis)); + return offset.localToUtc(roundedLocalMillis, this); + } + + @Override + public long inGap(long localMillis, Gap gap) { + // Round from just before the start of the gap + return gap.previous().localToUtc(unit.roundFloor(gap.firstMissingLocalTime() - 1), this); + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + return inGap(localMillis, gap); + } + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + // Convert the overlap at this offset because that'll produce the largest result. + return overlap.localToUtcInThisOffset(localMillis); + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + if (overlap.firstNonOverlappingLocalTime() - overlap.firstOverlappingLocalTime() >= unitMillis) { + return overlap.localToUtcInThisOffset(localMillis); + } + return overlap.previous().localToUtc(localMillis, this); // This is mostly for Asia/Lord_Howe + } + } + + private class JavaTimeToMidnightRounding implements Prepared { + @Override + public long round(long utcMillis) { + LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(utcMillis), timeZone); + LocalDateTime localMidnight = truncateLocalDateTime(localDateTime); + return firstTimeOnDay(localMidnight); + } + + @Override + public long nextRoundingValue(long utcMillis) { + LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(utcMillis), timeZone); + LocalDateTime earlierLocalMidnight = truncateLocalDateTime(localDateTime); + LocalDateTime localMidnight = nextRelevantMidnight(earlierLocalMidnight); + return firstTimeOnDay(localMidnight); + } + + private long firstTimeOnDay(LocalDateTime localMidnight) { + assert localMidnight.toLocalTime().equals(LocalTime.of(0, 0, 0)) : "firstTimeOnDay should only be called at midnight"; + + // Now work out what localMidnight actually means + final List currentOffsets = timeZone.getRules().getValidOffsets(localMidnight); + if (currentOffsets.isEmpty() == false) { + // There is at least one midnight on this day, so choose the first + final ZoneOffset firstOffset = currentOffsets.get(0); + final OffsetDateTime offsetMidnight = localMidnight.atOffset(firstOffset); + return offsetMidnight.toInstant().toEpochMilli(); + } else { + // There were no midnights on this day, so we must have entered the day via an offset transition. + // Use the time of the transition as it is the earliest time on the right day. + ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().getTransition(localMidnight); + return zoneOffsetTransition.getInstant().toEpochMilli(); + } + } + + private LocalDateTime nextRelevantMidnight(LocalDateTime localMidnight) { + assert localMidnight.toLocalTime().equals(LocalTime.MIDNIGHT) : "nextRelevantMidnight should only be called at midnight"; + + switch (unit) { + case DAY_OF_MONTH: + return localMidnight.plus(1, ChronoUnit.DAYS); + case WEEK_OF_WEEKYEAR: + return localMidnight.plus(7, ChronoUnit.DAYS); + case MONTH_OF_YEAR: + return localMidnight.plus(1, ChronoUnit.MONTHS); + case QUARTER_OF_YEAR: + return localMidnight.plus(3, ChronoUnit.MONTHS); + case YEAR_OF_CENTURY: + return localMidnight.plus(1, ChronoUnit.YEARS); + default: + throw new IllegalArgumentException("Unknown round-to-midnight unit: " + unit); + } + } + } + + private class JavaTimeNotToMidnightRounding extends AbstractNotToMidnightRounding { + JavaTimeNotToMidnightRounding(long unitMillis) { + super(unitMillis); + } + + @Override + public long round(long utcMillis) { + Instant instant = Instant.ofEpochMilli(utcMillis); + final ZoneRules rules = timeZone.getRules(); + while (true) { + final Instant truncatedTime = truncateAsLocalTime(instant, rules); + final ZoneOffsetTransition previousTransition = rules.previousTransition(instant); + + if (previousTransition == null) { + // truncateAsLocalTime cannot have failed if there were no previous transitions + return truncatedTime.toEpochMilli(); + } + + Instant previousTransitionInstant = previousTransition.getInstant(); + if (truncatedTime != null && previousTransitionInstant.compareTo(truncatedTime) < 1) { + return truncatedTime.toEpochMilli(); + } + + // There was a transition in between the input time and the truncated time. Return to the transition time and + // round that down instead. + instant = previousTransitionInstant.minusNanos(1_000_000); + } + } + + private Instant truncateAsLocalTime(Instant instant, final ZoneRules rules) { + assert unitRoundsToMidnight == false : "truncateAsLocalTime should not be called if unitRoundsToMidnight"; + + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, timeZone); + final LocalDateTime truncatedLocalDateTime = truncateLocalDateTime(localDateTime); + final List currentOffsets = rules.getValidOffsets(truncatedLocalDateTime); + + if (currentOffsets.isEmpty() == false) { + // at least one possibilities - choose the latest one that's still no later than the input time + for (int offsetIndex = currentOffsets.size() - 1; offsetIndex >= 0; offsetIndex--) { + final Instant result = truncatedLocalDateTime.atOffset(currentOffsets.get(offsetIndex)).toInstant(); + if (result.isAfter(instant) == false) { + return result; + } + } + + assert false : "rounded time not found for " + instant + " with " + this; + return null; + } else { + // The chosen local time didn't happen. This means we were given a time in an hour (or a minute) whose start + // is missing due to an offset transition, so the time cannot be truncated. + return null; + } + } + } + + private abstract class AbstractNotToMidnightRounding implements Prepared { + protected final long unitMillis; + + AbstractNotToMidnightRounding(long unitMillis) { + this.unitMillis = unitMillis; + } + + @Override + public final long nextRoundingValue(long utcMillis) { + final long roundedAfterOneIncrement = round(utcMillis + unitMillis); + if (utcMillis < roundedAfterOneIncrement) { + return roundedAfterOneIncrement; + } else { + return round(utcMillis + 2 * unitMillis); + } + } + } } static class TimeIntervalRounding extends Rounding { @@ -501,72 +772,89 @@ public byte id() { } @Override - public long round(final long utcMillis) { - // This works as long as the tz offset doesn't change. It is worth getting this case out of the way first, - // as the calculations for fixing things near to offset changes are a little expensive and unnecessary - // in the common case of working with fixed offset timezones (such as UTC). - if (fixedOffsetMillis != TZ_OFFSET_NON_FIXED) { - long localMillis = utcMillis + fixedOffsetMillis; - return (roundKey(localMillis, interval) * interval) - fixedOffsetMillis; - } - - final Instant utcInstant = Instant.ofEpochMilli(utcMillis); - final LocalDateTime rawLocalDateTime = LocalDateTime.ofInstant(utcInstant, timeZone); - - // a millisecond value with the same local time, in UTC, as `utcMillis` has in `timeZone` - final long localMillis = utcMillis + timeZone.getRules().getOffset(utcInstant).getTotalSeconds() * 1000; - assert localMillis == rawLocalDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); - - final long roundedMillis = roundKey(localMillis, interval) * interval; - final LocalDateTime roundedLocalDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(roundedMillis), ZoneOffset.UTC); - - // Now work out what roundedLocalDateTime actually means - final List currentOffsets = timeZone.getRules().getValidOffsets(roundedLocalDateTime); - if (currentOffsets.isEmpty() == false) { - // There is at least one instant with the desired local time. In general the desired result is - // the latest rounded time that's no later than the input time, but this could involve rounding across - // a timezone transition, which may yield the wrong result - final ZoneOffsetTransition previousTransition = timeZone.getRules().previousTransition(utcInstant.plusMillis(1)); - for (int offsetIndex = currentOffsets.size() - 1; 0 <= offsetIndex; offsetIndex--) { - final OffsetDateTime offsetTime = roundedLocalDateTime.atOffset(currentOffsets.get(offsetIndex)); - final Instant offsetInstant = offsetTime.toInstant(); - if (previousTransition != null && offsetInstant.isBefore(previousTransition.getInstant())) { - // Rounding down across the transition can yield the wrong result. It's best to return to the transition time - // and round that down. - return round(previousTransition.getInstant().toEpochMilli() - 1); - } - - if (utcInstant.isBefore(offsetTime.toInstant()) == false) { - return offsetInstant.toEpochMilli(); - } - } - - final OffsetDateTime offsetTime = roundedLocalDateTime.atOffset(currentOffsets.get(0)); - final Instant offsetInstant = offsetTime.toInstant(); - assert false : this + " failed to round " + utcMillis + " down: " + offsetInstant + " is the earliest possible"; - return offsetInstant.toEpochMilli(); // TODO or throw something? - } else { - // The desired time isn't valid because within a gap, so just return the gap time. - ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().getTransition(roundedLocalDateTime); - return zoneOffsetTransition.getInstant().toEpochMilli(); - } + public Prepared prepare(long minUtcMillis, long maxUtcMillis) { + return prepareForUnknown(); } - private static long roundKey(long value, long interval) { - if (value < 0) { - return (value - interval + 1) / interval; - } else { - return value / interval; - } + @Override + public Prepared prepareForUnknown() { + return prepareJavaTime(); } @Override - public long nextRoundingValue(long time) { - int offsetSeconds = timeZone.getRules().getOffset(Instant.ofEpochMilli(time)).getTotalSeconds(); - long millis = time + interval + offsetSeconds * 1000; - return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC) - .withZoneSameLocal(timeZone) - .toInstant().toEpochMilli(); + Prepared prepareJavaTime() { + return new Prepared() { + @Override + public long round(long utcMillis) { + if (fixedOffsetMillis != TZ_OFFSET_NON_FIXED) { + // This works as long as the tz offset doesn't change. It is worth getting this case out of the way first, + // as the calculations for fixing things near to offset changes are a little expensive and unnecessary + // in the common case of working with fixed offset timezones (such as UTC). + long localMillis = utcMillis + fixedOffsetMillis; + return (roundKey(localMillis, interval) * interval) - fixedOffsetMillis; + } + final Instant utcInstant = Instant.ofEpochMilli(utcMillis); + final LocalDateTime rawLocalDateTime = LocalDateTime.ofInstant(utcInstant, timeZone); + + // a millisecond value with the same local time, in UTC, as `utcMillis` has in `timeZone` + final long localMillis = utcMillis + timeZone.getRules().getOffset(utcInstant).getTotalSeconds() * 1000; + assert localMillis == rawLocalDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + + final long roundedMillis = roundKey(localMillis, interval) * interval; + final LocalDateTime roundedLocalDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(roundedMillis), ZoneOffset.UTC); + + // Now work out what roundedLocalDateTime actually means + final List currentOffsets = timeZone.getRules().getValidOffsets(roundedLocalDateTime); + if (currentOffsets.isEmpty() == false) { + // There is at least one instant with the desired local time. In general the desired result is + // the latest rounded time that's no later than the input time, but this could involve rounding across + // a timezone transition, which may yield the wrong result + final ZoneOffsetTransition previousTransition = timeZone.getRules().previousTransition(utcInstant.plusMillis(1)); + for (int offsetIndex = currentOffsets.size() - 1; 0 <= offsetIndex; offsetIndex--) { + final OffsetDateTime offsetTime = roundedLocalDateTime.atOffset(currentOffsets.get(offsetIndex)); + final Instant offsetInstant = offsetTime.toInstant(); + if (previousTransition != null && offsetInstant.isBefore(previousTransition.getInstant())) { + /* + * Rounding down across the transition can yield the + * wrong result. It's best to return to the transition + * time and round that down. + */ + return round(previousTransition.getInstant().toEpochMilli() - 1); + } + + if (utcInstant.isBefore(offsetTime.toInstant()) == false) { + return offsetInstant.toEpochMilli(); + } + } + + final OffsetDateTime offsetTime = roundedLocalDateTime.atOffset(currentOffsets.get(0)); + final Instant offsetInstant = offsetTime.toInstant(); + assert false : this + " failed to round " + utcMillis + " down: " + offsetInstant + " is the earliest possible"; + return offsetInstant.toEpochMilli(); // TODO or throw something? + } else { + // The desired time isn't valid because within a gap, so just return the gap time. + ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().getTransition(roundedLocalDateTime); + return zoneOffsetTransition.getInstant().toEpochMilli(); + } + } + + @Override + public long nextRoundingValue(long time) { + int offsetSeconds = timeZone.getRules().getOffset(Instant.ofEpochMilli(time)).getTotalSeconds(); + long millis = time + interval + offsetSeconds * 1000; + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC) + .withZoneSameLocal(timeZone) + .toInstant().toEpochMilli(); + } + + private long roundKey(long value, long interval) { + if (value < 0) { + return (value - interval + 1) / interval; + } else { + return value / interval; + } + } + }; } @Override @@ -634,13 +922,32 @@ public byte id() { } @Override - public long round(long value) { - return delegate.round(value - offset) + offset; + public Prepared prepare(long minUtcMillis, long maxUtcMillis) { + return wrapPreparedRounding(delegate.prepare(minUtcMillis, maxUtcMillis)); + } + + @Override + public Prepared prepareForUnknown() { + return wrapPreparedRounding(delegate.prepareForUnknown()); } @Override - public long nextRoundingValue(long value) { - return delegate.nextRoundingValue(value - offset) + offset; + Prepared prepareJavaTime() { + return wrapPreparedRounding(delegate.prepareJavaTime()); + } + + private Prepared wrapPreparedRounding(Prepared delegatePrepared) { + return new Prepared() { + @Override + public long round(long utcMillis) { + return delegatePrepared.round(utcMillis - offset) + offset; + } + + @Override + public long nextRoundingValue(long utcMillis) { + return delegatePrepared.nextRoundingValue(utcMillis - offset) + offset; + } + }; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index 1f61f4d0b0ee3..e3d6270da9e01 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -310,7 +310,7 @@ public static long toMilliSeconds(long nanoSecondsSinceEpoch) { * Rounds the given utc milliseconds sicne the epoch down to the next unit millis * * Note: This does not check for correctness of the result, as this only works with units smaller or equal than a day - * In order to ensure the performane of this methods, there are no guards or checks in it + * In order to ensure the performance of this methods, there are no guards or checks in it * * @param utcMillis the milliseconds since the epoch * @param unitMillis the unit to round to diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java index 805194b690ecb..13b1ad18feaf7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java @@ -19,11 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.composite; -import java.io.IOException; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.Objects; - import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Rounding; @@ -46,6 +41,11 @@ import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Objects; + /** * A {@link CompositeValuesSourceBuilder} that builds a {@link RoundingValuesSource} from a {@link Script} or * a field name using the provided interval. @@ -255,7 +255,9 @@ protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardCon } if (orig instanceof ValuesSource.Numeric) { ValuesSource.Numeric numeric = (ValuesSource.Numeric) orig; - RoundingValuesSource vs = new RoundingValuesSource(numeric, rounding); + // TODO once composite is plugged in to the values source registry or at least understands Date values source types use it here + Rounding.Prepared preparedRounding = rounding.prepareForUnknown(); + RoundingValuesSource vs = new RoundingValuesSource(numeric, preparedRounding); // is specified in the builder. final DocValueFormat docValueFormat = format() == null ? DocValueFormat.RAW : config.format(); final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java index 9ee142fcd2fd5..fec144fcc5c2a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java @@ -34,14 +34,14 @@ */ class RoundingValuesSource extends ValuesSource.Numeric { private final ValuesSource.Numeric vs; - private final Rounding rounding; + private final Rounding.Prepared rounding; /** * * @param vs The original values source * @param rounding How to round the values */ - RoundingValuesSource(Numeric vs, Rounding rounding) { + RoundingValuesSource(Numeric vs, Rounding.Prepared rounding) { this.vs = vs; this.rounding = rounding; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java index 6be20aa6fece6..30c17846e833a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java @@ -45,33 +45,37 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; /** - * An aggregator for date values. Every date is rounded down using a configured - * {@link Rounding}. - * - * @see Rounding + * An aggregator for date values that attempts to return a specific number of + * buckets, reconfiguring how it rounds dates to buckets on the fly as new + * data arrives. */ class AutoDateHistogramAggregator extends DeferableBucketAggregator { private final ValuesSource.Numeric valuesSource; private final DocValueFormat formatter; private final RoundingInfo[] roundingInfos; + private final Function roundingPreparer; private int roundingIdx = 0; + private Rounding.Prepared preparedRounding; private LongHash bucketOrds; private int targetBuckets; private MergingBucketsDeferringCollector deferringCollector; AutoDateHistogramAggregator(String name, AggregatorFactories factories, int numBuckets, RoundingInfo[] roundingInfos, - @Nullable ValuesSource valuesSource, DocValueFormat formatter, SearchContext aggregationContext, Aggregator parent, - Map metadata) throws IOException { + Function roundingPreparer, @Nullable ValuesSource valuesSource, DocValueFormat formatter, + SearchContext aggregationContext, Aggregator parent, Map metadata) throws IOException { super(name, factories, aggregationContext, parent, metadata); this.targetBuckets = numBuckets; this.valuesSource = (ValuesSource.Numeric) valuesSource; this.formatter = formatter; this.roundingInfos = roundingInfos; + this.roundingPreparer = roundingPreparer; + preparedRounding = roundingPreparer.apply(roundingInfos[roundingIdx].rounding); bucketOrds = new LongHash(1, aggregationContext.bigArrays()); @@ -113,7 +117,7 @@ public void collect(int doc, long bucket) throws IOException { long previousRounded = Long.MIN_VALUE; for (int i = 0; i < valuesCount; ++i) { long value = values.nextValue(); - long rounded = roundingInfos[roundingIdx].rounding.round(value); + long rounded = preparedRounding.round(value); assert rounded >= previousRounded; if (rounded == previousRounded) { continue; @@ -138,10 +142,10 @@ private void increaseRounding() { try (LongHash oldBucketOrds = bucketOrds) { LongHash newBucketOrds = new LongHash(1, context.bigArrays()); long[] mergeMap = new long[(int) oldBucketOrds.size()]; - Rounding newRounding = roundingInfos[++roundingIdx].rounding; + preparedRounding = roundingPreparer.apply(roundingInfos[++roundingIdx].rounding); for (int i = 0; i < oldBucketOrds.size(); i++) { long oldKey = oldBucketOrds.get(i); - long newKey = newRounding.round(oldKey); + long newKey = preparedRounding.round(oldKey); long newBucketOrd = newBucketOrds.add(newKey); if (newBucketOrd >= 0) { mergeMap[i] = newBucketOrd; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java index 7f5b856aaa488..58c6f1944956a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.histogram; +import org.elasticsearch.common.Rounding; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; @@ -76,15 +77,16 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, throw new AggregationExecutionException("Registry miss-match - expected AutoDateHistogramAggregationSupplier, found [" + aggregatorSupplier.getClass().toString() + "]"); } - return ((AutoDateHistogramAggregatorSupplier) aggregatorSupplier).build(name, factories, numBuckets, roundingInfos, valuesSource, - config.format(), searchContext, parent, metadata); + return ((AutoDateHistogramAggregatorSupplier) aggregatorSupplier).build(name, factories, numBuckets, roundingInfos, + // TODO once auto date histo is plugged into the ValuesSource refactoring use the date values source + Rounding::prepareForUnknown, valuesSource, config.format(), searchContext, parent, metadata); } @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { - return new AutoDateHistogramAggregator(name, factories, numBuckets, roundingInfos, null, config.format(), searchContext, parent, - metadata); + return new AutoDateHistogramAggregator(name, factories, numBuckets, roundingInfos, Rounding::prepareForUnknown, null, + config.format(), searchContext, parent, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java index 77ccc552d85ab..e5c286f431c1b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Rounding; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -29,6 +30,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.Function; @FunctionalInterface public interface AutoDateHistogramAggregatorSupplier extends AggregatorSupplier { @@ -37,6 +39,8 @@ Aggregator build( AggregatorFactories factories, int numBuckets, AutoDateHistogramAggregationBuilder.RoundingInfo[] roundingInfos, + @Nullable + Function roundingPreparer, @Nullable ValuesSource valuesSource, DocValueFormat formatter, SearchContext aggregationContext, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java index ebeeb855bea25..99b352303770e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java @@ -523,6 +523,7 @@ protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardC AggregatorFactories.Builder subFactoriesBuilder) throws IOException { final ZoneId tz = timeZone(); final Rounding rounding = dateHistogramInterval.createRounding(tz, offset); + // TODO once we optimize TimeIntervalRounding we won't need to rewrite the time zone final ZoneId rewrittenTimeZone = rewriteTimeZone(queryShardContext); final Rounding shardRounding; if (tz == rewrittenTimeZone) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java index 623f75204e696..70c6028e69a25 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java @@ -37,7 +37,7 @@ public interface DateHistogramAggregationSupplier extends AggregatorSupplier { Aggregator build(String name, AggregatorFactories factories, Rounding rounding, - Rounding shardRounding, + Rounding.Prepared preparedRounding, BucketOrder order, boolean keyed, long minDocCount, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java index 5fe2f31090468..c997c7d50467a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java @@ -54,7 +54,10 @@ class DateHistogramAggregator extends BucketsAggregator { private final ValuesSource.Numeric valuesSource; private final DocValueFormat formatter; private final Rounding rounding; - private final Rounding shardRounding; + /** + * The rounding prepared for rewriting the data in the shard. + */ + private final Rounding.Prepared preparedRounding; private final BucketOrder order; private final boolean keyed; @@ -63,21 +66,21 @@ class DateHistogramAggregator extends BucketsAggregator { private final LongHash bucketOrds; - DateHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, Rounding shardRounding, + DateHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, Rounding.Prepared preparedRounding, BucketOrder order, boolean keyed, - long minDocCount, @Nullable ExtendedBounds extendedBounds, @Nullable ValuesSource.Numeric valuesSource, + long minDocCount, @Nullable ExtendedBounds extendedBounds, @Nullable ValuesSource valuesSource, DocValueFormat formatter, SearchContext aggregationContext, Aggregator parent, Map metadata) throws IOException { super(name, factories, aggregationContext, parent, metadata); this.rounding = rounding; - this.shardRounding = shardRounding; + this.preparedRounding = preparedRounding; this.order = order; order.validate(this); this.keyed = keyed; this.minDocCount = minDocCount; this.extendedBounds = extendedBounds; - this.valuesSource = valuesSource; + this.valuesSource = (ValuesSource.Numeric) valuesSource; this.formatter = formatter; bucketOrds = new LongHash(1, aggregationContext.bigArrays()); @@ -110,7 +113,7 @@ public void collect(int doc, long bucket) throws IOException { long value = values.nextValue(); // We can use shardRounding here, which is sometimes more efficient // if daylight saving times are involved. - long rounded = shardRounding.round(value); + long rounded = preparedRounding.round(value); assert rounded >= previousRounded; if (rounded == previousRounded) { continue; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java index 44e9736f5360b..884e44b405ca4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java @@ -19,11 +19,8 @@ package org.elasticsearch.search.aggregations.bucket.histogram; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Rounding; -import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -46,46 +43,11 @@ public final class DateHistogramAggregatorFactory extends ValuesSourceAggregator public static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register(DateHistogramAggregationBuilder.NAME, List.of(CoreValuesSourceType.DATE, CoreValuesSourceType.NUMERIC, CoreValuesSourceType.BOOLEAN), - (DateHistogramAggregationSupplier) (String name, - AggregatorFactories factories, - Rounding rounding, - Rounding shardRounding, - BucketOrder order, - boolean keyed, - long minDocCount, - @Nullable ExtendedBounds extendedBounds, - @Nullable ValuesSource valuesSource, - DocValueFormat formatter, - SearchContext aggregationContext, - Aggregator parent, - Map metadata) -> new DateHistogramAggregator(name, - factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, (ValuesSource.Numeric) valuesSource, - formatter, aggregationContext, parent, metadata)); + (DateHistogramAggregationSupplier) DateHistogramAggregator::new); builder.register(DateHistogramAggregationBuilder.NAME, CoreValuesSourceType.RANGE, - (DateHistogramAggregationSupplier) (String name, - AggregatorFactories factories, - Rounding rounding, - Rounding shardRounding, - BucketOrder order, - boolean keyed, - long minDocCount, - @Nullable ExtendedBounds extendedBounds, - @Nullable ValuesSource valuesSource, - DocValueFormat formatter, - SearchContext aggregationContext, - Aggregator parent, - Map metadata) -> { - - ValuesSource.Range rangeValueSource = (ValuesSource.Range) valuesSource; - if (rangeValueSource.rangeType() != RangeType.DATE) { - throw new IllegalArgumentException("Expected date range type but found range type [" + rangeValueSource.rangeType().name - + "]"); - } - return new DateRangeHistogramAggregator(name, - factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, rangeValueSource, formatter, - aggregationContext, parent, metadata); }); + (DateHistogramAggregationSupplier) DateRangeHistogramAggregator::new); } private final BucketOrder order; @@ -128,15 +90,17 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, throw new AggregationExecutionException("Registry miss-match - expected DateHistogramAggregationSupplier, found [" + aggregatorSupplier.getClass().toString() + "]"); } - return ((DateHistogramAggregationSupplier) aggregatorSupplier).build(name, factories, rounding, shardRounding, order, keyed, - minDocCount, extendedBounds, valuesSource, config.format(), searchContext, parent, metadata); + Rounding.Prepared preparedRounding = valuesSource.roundingPreparer(queryShardContext.getIndexReader()).apply(shardRounding); + return ((DateHistogramAggregationSupplier) aggregatorSupplier).build(name, factories, rounding, preparedRounding, order, keyed, + minDocCount, extendedBounds, valuesSource, config.format(), searchContext, + parent, metadata); } @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { - return new DateHistogramAggregator(name, factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, + return new DateHistogramAggregator(name, factories, rounding, null, order, keyed, minDocCount, extendedBounds, null, config.format(), searchContext, parent, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java index c26db7ef3748d..97333db3d85eb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java @@ -57,7 +57,10 @@ class DateRangeHistogramAggregator extends BucketsAggregator { private final ValuesSource.Range valuesSource; private final DocValueFormat formatter; private final Rounding rounding; - private final Rounding shardRounding; + /** + * The rounding prepared for rewriting the data in the shard. + */ + private final Rounding.Prepared preparedRounding; private final BucketOrder order; private final boolean keyed; @@ -66,22 +69,26 @@ class DateRangeHistogramAggregator extends BucketsAggregator { private final LongHash bucketOrds; - DateRangeHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, Rounding shardRounding, + DateRangeHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, Rounding.Prepared preparedRounding, BucketOrder order, boolean keyed, - long minDocCount, @Nullable ExtendedBounds extendedBounds, @Nullable ValuesSource.Range valuesSource, + long minDocCount, @Nullable ExtendedBounds extendedBounds, @Nullable ValuesSource valuesSource, DocValueFormat formatter, SearchContext aggregationContext, Aggregator parent, Map metadata) throws IOException { super(name, factories, aggregationContext, parent, metadata); this.rounding = rounding; - this.shardRounding = shardRounding; + this.preparedRounding = preparedRounding; this.order = order; order.validate(this); this.keyed = keyed; this.minDocCount = minDocCount; this.extendedBounds = extendedBounds; - this.valuesSource = valuesSource; + this.valuesSource = (ValuesSource.Range) valuesSource; this.formatter = formatter; + if (this.valuesSource.rangeType() != RangeType.DATE) { + throw new IllegalArgumentException("Expected date range type but found range type [" + this.valuesSource.rangeType().name + + "]"); + } bucketOrds = new LongHash(1, aggregationContext.bigArrays()); } @@ -122,10 +129,10 @@ public void collect(int doc, long bucket) throws IOException { // The encoding should ensure that this assert is always true. assert from >= previousFrom : "Start of range not >= previous start"; final Long to = (Long) range.getTo(); - final long startKey = shardRounding.round(from); - final long endKey = shardRounding.round(to); + final long startKey = preparedRounding.round(from); + final long endKey = preparedRounding.round(to); for (long key = startKey > previousKey ? startKey : previousKey; key <= endKey; - key = shardRounding.nextRoundingValue(key)) { + key = preparedRounding.nextRoundingValue(key)) { if (key == previousKey) { continue; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index fffda04900e57..1b79bd7925f95 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -19,7 +19,11 @@ package org.elasticsearch.search.aggregations.support; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.PointValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Rounding; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -27,17 +31,20 @@ import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.script.AggregationScript; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; +import java.io.IOException; import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.function.Function; import java.util.function.LongSupplier; /** @@ -235,7 +242,51 @@ public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType sc @Override public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - return NUMERIC.getField(fieldContext, script); + ValuesSource.Numeric dataSource = fieldData(fieldContext); + if (script != null) { + // Value script case + return new ValuesSource.Numeric.WithScript(dataSource, script); + } + return dataSource; + } + + private ValuesSource.Numeric fieldData(FieldContext fieldContext) { + if ((fieldContext.indexFieldData() instanceof IndexNumericFieldData) == false) { + throw new IllegalArgumentException("Expected numeric type on field [" + fieldContext.field() + + "], but got [" + fieldContext.fieldType().typeName() + "]"); + } + if (fieldContext.fieldType().indexOptions() == IndexOptions.NONE + || fieldContext.fieldType() instanceof DateFieldType == false) { + /* + * We can't implement roundingPreparer in these cases because + * we can't look up the min and max date without both the + * search index (the first test) and the resolution which is + * on the DateFieldType. + */ + return new ValuesSource.Numeric.FieldData((IndexNumericFieldData) fieldContext.indexFieldData()); + } + return new ValuesSource.Numeric.FieldData((IndexNumericFieldData) fieldContext.indexFieldData()) { + /** + * Proper dates get a real implementation of + * {@link #roundingPreparer(IndexReader)}. If the field is + * configured with a script or a missing value then we'll + * wrap this without delegating so those fields will ignore + * this implementation. Which is correct. + */ + @Override + public Function roundingPreparer(IndexReader reader) throws IOException { + DateFieldType dft = (DateFieldType) fieldContext.fieldType(); + byte[] min = PointValues.getMinPackedValue(reader, fieldContext.field()); + if (min == null) { + // There aren't any indexes values so we don't need to optimize. + return Rounding::prepareForUnknown; + } + byte[] max = PointValues.getMaxPackedValue(reader, fieldContext.field()); + long minUtcMillis = dft.resolution().parsePointAsMillis(min); + long maxUtcMillis = dft.resolution().parsePointAsMillis(max); + return rounding -> rounding.prepare(minUtcMillis, maxUtcMillis); + } + }; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/FieldContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/FieldContext.java index a88730e582f42..f90cab1b5fcd0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/FieldContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/FieldContext.java @@ -23,7 +23,7 @@ /** * Used by all field data based aggregators. This determine the context of the field data the aggregators are operating - * in. I holds both the field names and the index field datas that are associated with them. + * in. It holds both the field names and the index field datas that are associated with them. */ public class FieldContext { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 1b1552bf0336e..62d2e32c0b8b7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -28,15 +28,17 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorable; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.Rounding.Prepared; import org.elasticsearch.common.lucene.ScorerAware; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; -import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.DocValueBits; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -44,12 +46,14 @@ import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithScript.BytesValues; import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues; import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues; import org.elasticsearch.search.aggregations.support.values.ScriptLongValues; import java.io.IOException; +import java.util.function.Function; import java.util.function.LongUnaryOperator; /** @@ -74,6 +78,14 @@ public boolean needsScores() { return false; } + /** + * Build a function prepares rounding values to be called many times. + *

+ * This returns a {@linkplain Function} because auto date histogram will + * need to call it many times over the course of running the aggregation. + */ + public abstract Function roundingPreparer(IndexReader reader) throws IOException; + public static class Range extends ValuesSource { private final RangeType rangeType; protected final IndexFieldData indexFieldData; @@ -94,6 +106,12 @@ public DocValueBits docsWithValue(LeafReaderContext context) throws IOException return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes); } + @Override + public Function roundingPreparer(IndexReader reader) throws IOException { + // TODO lookup the min and max rounding when appropriate + return Rounding::prepareForUnknown; + } + public RangeType rangeType() { return rangeType; } } public abstract static class Bytes extends ValuesSource { @@ -104,6 +122,11 @@ public DocValueBits docsWithValue(LeafReaderContext context) throws IOException return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes); } + @Override + public final Function roundingPreparer(IndexReader reader) throws IOException { + throw new AggregationExecutionException("can't round a [BYTES]"); + } + public abstract static class WithOrdinals extends Bytes { public static final WithOrdinals EMPTY = new WithOrdinals() { @@ -359,6 +382,11 @@ public DocValueBits docsWithValue(LeafReaderContext context) throws IOException } } + @Override + public Function roundingPreparer(IndexReader reader) throws IOException { + return Rounding::prepareForUnknown; + } + /** * {@link ValuesSource} subclass for Numeric fields with a Value Script applied */ @@ -551,6 +579,11 @@ public DocValueBits docsWithValue(LeafReaderContext context) throws IOException return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints); } + @Override + public final Function roundingPreparer(IndexReader reader) throws IOException { + throw new AggregationExecutionException("can't round a [GEO_POINT]"); + } + public abstract MultiGeoPointValues geoPointValues(LeafReaderContext context); public static class Fielddata extends GeoPoint { diff --git a/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java b/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java new file mode 100644 index 0000000000000..b5f79b0efd1ef --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java @@ -0,0 +1,380 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import org.elasticsearch.common.LocalTimeOffset.Gap; +import org.elasticsearch.common.LocalTimeOffset.Overlap; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.test.ESTestCase; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.zone.ZoneOffsetTransition; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class LocalTimeOffsetTests extends ESTestCase { + public void testRangeTooLarge() { + ZoneId zone = ZoneId.of("America/New_York"); + assertThat(LocalTimeOffset.lookup(zone, Long.MIN_VALUE, Long.MAX_VALUE), nullValue()); + } + + public void testNotFixed() { + ZoneId zone = ZoneId.of("America/New_York"); + assertThat(LocalTimeOffset.lookupFixedOffset(zone), nullValue()); + } + + public void testUtc() { + assertFixOffset(ZoneId.of("UTC"), 0); + } + + public void testFixedOffset() { + ZoneOffset zone = ZoneOffset.ofTotalSeconds(between((int) -TimeUnit.HOURS.toSeconds(18), (int) TimeUnit.HOURS.toSeconds(18))); + assertFixOffset(zone, zone.getTotalSeconds() * 1000); + } + + private void assertFixOffset(ZoneId zone, long offsetMillis) { + LocalTimeOffset fixed = LocalTimeOffset.lookupFixedOffset(zone); + assertThat(fixed, notNullValue()); + + LocalTimeOffset.Lookup lookup = LocalTimeOffset.lookup(zone, Long.MIN_VALUE, Long.MAX_VALUE); + assertThat(lookup.size(), equalTo(1)); + long min = randomLong(); + long max = randomValueOtherThan(min, ESTestCase::randomLong); + if (min > max) { + long s = min; + min = max; + max = s; + } + LocalTimeOffset fixedInRange = lookup.fixedInRange(min, max); + assertThat(fixedInRange, notNullValue()); + + assertRoundingAtOffset(randomBoolean() ? fixed : fixedInRange, randomLong(), offsetMillis); + } + + private void assertRoundingAtOffset(LocalTimeOffset offset, long time, long offsetMillis) { + assertThat(offset.utcToLocalTime(time), equalTo(time + offsetMillis)); + assertThat(offset.localToUtcInThisOffset(time + offsetMillis), equalTo(time)); + assertThat(offset.localToUtc(time + offsetMillis, unusedStrategy()), equalTo(time)); + } + + public void testJustTransitions() { + ZoneId zone = ZoneId.of("America/New_York"); + long min = time("1980-01-01", zone); + long max = time("1981-01-01", zone) - 1; + assertThat(Instant.ofEpochMilli(max), lessThan(lastTransitionIn(zone).getInstant())); + assertTransitions(zone, min, max, time("1980-06-01", zone), min + hours(1), 3, hours(-5), hours(-4)); + } + + public void testTransitionsWithTransitionsAndRules() { + ZoneId zone = ZoneId.of("America/New_York"); + long min = time("1980-01-01", zone); + long max = time("2021-01-01", zone) - 1; + assertThat(Instant.ofEpochMilli(min), lessThan(lastTransitionIn(zone).getInstant())); + assertThat(Instant.ofEpochMilli(max), greaterThan(lastTransitionIn(zone).getInstant())); + assertTransitions(zone, min, max, time("2000-06-01", zone), min + hours(1), 83, hours(-5), hours(-4)); + assertThat(LocalTimeOffset.lookup(zone, min, max).fixedInRange(utcTime("2000-06-01"), utcTime("2000-06-02")), notNullValue()); + } + + public void testAfterRules() { + ZoneId zone = ZoneId.of("America/New_York"); + long min = time("2020-01-01", zone); + long max = time("2021-01-01", zone) - 1; + assertThat(Instant.ofEpochMilli(min), greaterThan(lastTransitionIn(zone).getInstant())); + assertTransitions(zone, min, max, time("2020-06-01", zone), min + hours(1), 3, hours(-5), hours(-4)); + } + + private void assertTransitions(ZoneId zone, long min, long max, long between, long sameOffsetAsMin, + int size, long minMaxOffset, long betweenOffset) { + LocalTimeOffset.Lookup lookup = LocalTimeOffset.lookup(zone, min, max); + assertThat(lookup.size(), equalTo(size)); + assertRoundingAtOffset(lookup.lookup(min), min, minMaxOffset); + assertRoundingAtOffset(lookup.lookup(between), between, betweenOffset); + assertRoundingAtOffset(lookup.lookup(max), max, minMaxOffset); + assertThat(lookup.fixedInRange(min, max), nullValue()); + assertThat(lookup.fixedInRange(min, sameOffsetAsMin), sameInstance(lookup.lookup(min))); + } + + // Some sanity checks for when you pas a single time. We don't expect to do this much but it shouldn't be totally borked. + public void testSingleTimeBeforeRules() { + ZoneId zone = ZoneId.of("America/New_York"); + long time = time("1980-01-01", zone); + assertThat(Instant.ofEpochMilli(time), lessThan(lastTransitionIn(zone).getInstant())); + assertRoundingAtOffset(LocalTimeOffset.lookup(zone, time, time).lookup(time), time, hours(-5)); + } + + public void testSingleTimeAfterRules() { + ZoneId zone = ZoneId.of("America/New_York"); + long time = time("2020-01-01", zone); + assertThat(Instant.ofEpochMilli(time), greaterThan(lastTransitionIn(zone).getInstant())); + assertRoundingAtOffset(LocalTimeOffset.lookup(zone, time, time).lookup(time), time, hours(-5)); + } + + public void testJustOneRuleApplies() { + ZoneId zone = ZoneId.of("Atlantic/Azores"); + long time = time("2000-10-30T00:00:00", zone); + assertRoundingAtOffset(LocalTimeOffset.lookup(zone, time, time).lookup(time), time, hours(-1)); + } + + public void testLastTransitionWithoutRules() { + /* + * Asia/Kathmandu turned their clocks 15 minutes forward at + * 1986-01-01T00:00:00 local time and hasn't changed time since. + * This has broken the transition collection code in the past. + */ + ZoneId zone = ZoneId.of("Asia/Kathmandu"); + long time = time("1986-01-01T00:00:00", zone); + LocalTimeOffset.Lookup lookup = LocalTimeOffset.lookup(zone, time - 1, time); + assertThat(lookup.size(), equalTo(2)); + assertRoundingAtOffset(lookup.lookup(time - 1), time - 1, TimeUnit.MINUTES.toMillis(330)); + assertRoundingAtOffset(lookup.lookup(time), time, TimeUnit.MINUTES.toMillis(345)); + } + + public void testOverlap() { + /* + * Europe/Rome turn their clocks back an hour 1978 which is totally + * normal, but they rolled back past midnight which is pretty rare and neat. + */ + ZoneId tz = ZoneId.of("Europe/Rome"); + long overlapMillis = TimeUnit.HOURS.toMillis(1); + long firstMidnight = utcTime("1978-09-30T22:00:00"); + long secondMidnight = utcTime("1978-09-30T23:00:00"); + long overlapEnds = utcTime("1978-10-01T0:00:00"); + LocalTimeOffset.Lookup lookup = LocalTimeOffset.lookup(tz, firstMidnight, overlapEnds); + LocalTimeOffset secondMidnightOffset = lookup.lookup(secondMidnight); + long localSecondMidnight = secondMidnightOffset.utcToLocalTime(secondMidnight); + LocalTimeOffset firstMidnightOffset = lookup.lookup(firstMidnight); + long localFirstMidnight = firstMidnightOffset.utcToLocalTime(firstMidnight); + assertThat(localSecondMidnight - localFirstMidnight, equalTo(0L)); + assertThat(lookup.lookup(overlapEnds), sameInstance(secondMidnightOffset)); + long localOverlapEnds = secondMidnightOffset.utcToLocalTime(overlapEnds); + assertThat(localOverlapEnds - localSecondMidnight, equalTo(overlapMillis)); + + long localOverlappingTime = randomLongBetween(localFirstMidnight, localOverlapEnds); + + assertThat(firstMidnightOffset.localToUtcInThisOffset(localFirstMidnight - 1), equalTo(firstMidnight - 1)); + assertThat(secondMidnightOffset.localToUtcInThisOffset(localFirstMidnight - 1), equalTo(secondMidnight - 1)); + assertThat(firstMidnightOffset.localToUtcInThisOffset(localFirstMidnight), equalTo(firstMidnight)); + assertThat(secondMidnightOffset.localToUtcInThisOffset(localFirstMidnight), equalTo(secondMidnight)); + assertThat(secondMidnightOffset.localToUtcInThisOffset(localOverlapEnds), equalTo(overlapEnds)); + assertThat(secondMidnightOffset.localToUtcInThisOffset(localOverlappingTime), + equalTo(firstMidnightOffset.localToUtcInThisOffset(localOverlappingTime) + overlapMillis)); + + long beforeOverlapValue = randomLong(); + assertThat(secondMidnightOffset.localToUtc(localFirstMidnight - 1, useValueForBeforeOverlap(beforeOverlapValue)), + equalTo(beforeOverlapValue)); + long overlapValue = randomLong(); + assertThat(secondMidnightOffset.localToUtc(localFirstMidnight, useValueForOverlap(overlapValue)), equalTo(overlapValue)); + assertThat(secondMidnightOffset.localToUtc(localOverlapEnds, unusedStrategy()), equalTo(overlapEnds)); + assertThat(secondMidnightOffset.localToUtc(localOverlappingTime, useValueForOverlap(overlapValue)), equalTo(overlapValue)); + } + + public void testGap() { + /* + * Asia/Kathmandu turned their clocks 15 minutes forward at + * 1986-01-01T00:00:00, creating a really "fun" gap. + */ + ZoneId tz = ZoneId.of("Asia/Kathmandu"); + long gapLength = TimeUnit.MINUTES.toMillis(15); + long transition = time("1986-01-01T00:00:00", tz); + LocalTimeOffset.Lookup lookup = LocalTimeOffset.lookup(tz, transition - 1, transition); + LocalTimeOffset gapOffset = lookup.lookup(transition); + long localAtTransition = gapOffset.utcToLocalTime(transition); + LocalTimeOffset beforeGapOffset = lookup.lookup(transition - 1); + long localBeforeTransition = beforeGapOffset.utcToLocalTime(transition - 1); + assertThat(localAtTransition - localBeforeTransition, equalTo(gapLength + 1)); + + assertThat(beforeGapOffset.localToUtcInThisOffset(localBeforeTransition), equalTo(transition - 1)); + assertThat(gapOffset.localToUtcInThisOffset(localBeforeTransition), equalTo(transition - 1 - gapLength)); + assertThat(gapOffset.localToUtcInThisOffset(localAtTransition), equalTo(transition)); + + long beforeGapValue = randomLong(); + assertThat(gapOffset.localToUtc(localBeforeTransition, useValueForBeforeGap(beforeGapValue)), equalTo(beforeGapValue)); + assertThat(gapOffset.localToUtc(localAtTransition, unusedStrategy()), equalTo(transition)); + long gapValue = randomLong(); + long localSkippedTime = randomLongBetween(localBeforeTransition, localAtTransition); + assertThat(gapOffset.localToUtc(localSkippedTime, useValueForGap(gapValue)), equalTo(gapValue)); + } + + private static long utcTime(String time) { + return DateFormatter.forPattern("date_optional_time").parseMillis(time); + } + + private static long time(String time, ZoneId zone) { + return DateFormatter.forPattern("date_optional_time").withZone(zone).parseMillis(time); + } + + /** + * The the last "fully defined" transitions in the provided {@linkplain ZoneId}. + */ + private static ZoneOffsetTransition lastTransitionIn(ZoneId zone) { + List transitions = zone.getRules().getTransitions(); + return transitions.get(transitions.size() -1); + } + + private static LocalTimeOffset.Strategy unusedStrategy() { + return new LocalTimeOffset.Strategy() { + @Override + public long inGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + }; + } + + private static LocalTimeOffset.Strategy useValueForGap(long gapValue) { + return new LocalTimeOffset.Strategy() { + @Override + public long inGap(long localMillis, Gap gap) { + return gapValue; + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + }; + } + + private static LocalTimeOffset.Strategy useValueForBeforeGap(long beforeGapValue) { + return new LocalTimeOffset.Strategy() { + @Override + public long inGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + return beforeGapValue; + } + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + }; + } + + private static LocalTimeOffset.Strategy useValueForOverlap(long overlapValue) { + return new LocalTimeOffset.Strategy() { + @Override + public long inGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + return overlapValue; + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + }; + } + + + private static LocalTimeOffset.Strategy useValueForBeforeOverlap(long beforeOverlapValue) { + return new LocalTimeOffset.Strategy() { + @Override + public long inGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeGap(long localMillis, Gap gap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long inOverlap(long localMillis, Overlap overlap) { + fail("Shouldn't be called"); + return 0; + } + + @Override + public long beforeOverlap(long localMillis, Overlap overlap) { + return beforeOverlapValue; + } + }; + } + + private static long hours(long hours) { + return TimeUnit.HOURS.toMillis(hours); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java index 64537b2b008b8..fa0df85e8648f 100644 --- a/server/src/test/java/org/elasticsearch/common/RoundingTests.java +++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java @@ -226,19 +226,23 @@ public void testOffsetRounding() { * described in * {@link #assertInterval(long, long, long, Rounding, ZoneId)} */ - public void testRoundingRandom() { + public void testRandomTimeUnitRounding() { for (int i = 0; i < 1000; ++i) { Rounding.DateTimeUnit unit = randomFrom(Rounding.DateTimeUnit.values()); ZoneId tz = randomZone(); Rounding rounding = new Rounding.TimeUnitRounding(unit, tz); - long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00 + long[] bounds = randomDateBounds(); + Rounding.Prepared prepared = rounding.prepare(bounds[0], bounds[1]); + + // Check that rounding is internally consistent and consistent with nextRoundingValue + long date = dateBetween(bounds[0], bounds[1]); long unitMillis = unit.getField().getBaseUnit().getDuration().toMillis(); // FIXME this was copy pasted from the other impl and not used. breaks the nasty date actually gets assigned if (randomBoolean()) { nastyDate(date, tz, unitMillis); } - final long roundedDate = rounding.round(date); - final long nextRoundingValue = rounding.nextRoundingValue(roundedDate); + final long roundedDate = prepared.round(date); + final long nextRoundingValue = prepared.nextRoundingValue(roundedDate); assertInterval(roundedDate, date, nextRoundingValue, rounding, tz); @@ -252,6 +256,26 @@ public void testRoundingRandom() { + Instant.ofEpochMilli(roundedDate), nextRoundingValue - roundedDate, equalTo(unitMillis)); } } + + // Round a whole bunch of dates and make sure they line up with the known good java time implementation + Rounding.Prepared javaTimeRounding = rounding.prepareJavaTime(); + for (int d = 0; d < 1000; d++) { + date = dateBetween(bounds[0], bounds[1]); + long javaRounded = javaTimeRounding.round(date); + long esRounded = prepared.round(date); + if (javaRounded != esRounded) { + fail("Expected [" + rounding + "] to round [" + Instant.ofEpochMilli(date) + "] to [" + + Instant.ofEpochMilli(javaRounded) + "] but instead rounded to [" + Instant.ofEpochMilli(esRounded) + "]"); + } + long javaNextRoundingValue = javaTimeRounding.nextRoundingValue(date); + long esNextRoundingValue = prepared.nextRoundingValue(date); + if (javaNextRoundingValue != esNextRoundingValue) { + fail("Expected [" + rounding + "] to round [" + Instant.ofEpochMilli(date) + "] to [" + + Instant.ofEpochMilli(esRounded) + "] and nextRoundingValue to be [" + + Instant.ofEpochMilli(javaNextRoundingValue) + "] but instead was to [" + + Instant.ofEpochMilli(esNextRoundingValue) + "]"); + } + } } } @@ -361,10 +385,7 @@ public void testIntervalRounding_HalfDay_DST() { assertThat(rounding.round(time("2016-03-28T13:00:00+02:00")), isDate(time("2016-03-28T12:00:00+02:00"), tz)); } - /** - * randomized test on {@link org.elasticsearch.common.Rounding.TimeIntervalRounding} with random interval and time zone offsets - */ - public void testIntervalRoundingRandom() { + public void testRandomTimeIntervalRounding() { for (int i = 0; i < 1000; i++) { TimeUnit unit = randomFrom(TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS); long interval = unit.toMillis(randomIntBetween(1, 365)); @@ -703,6 +724,60 @@ public void testDST_END_Edgecases() { assertInterval(midnightAfterTransition, nextMidnight, rounding, 24 * 60, tz); } + public void testBeforeOverlapLarge() { + // Moncton has a perfectly normal hour long Daylight Savings time. + ZoneId tz = ZoneId.of("America/Moncton"); + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build(); + assertThat(rounding.round(time("2003-10-26T03:43:35.079Z")), isDate(time("2003-10-26T03:00:00Z"), tz)); + } + + public void testBeforeOverlapSmall() { + /* + * Lord Howe is fun because Daylight Savings time is only 30 minutes + * so we round HOUR_OF_DAY differently. + */ + ZoneId tz = ZoneId.of("Australia/Lord_Howe"); + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build(); + assertThat(rounding.round(time("2018-03-31T15:25:15.148Z")), isDate(time("2018-03-31T14:00:00Z"), tz)); + } + + public void testQuarterOfYear() { + /* + * If we're not careful with how we look up local time offsets we can + * end up not loading the offsets far enough back to round this time + * to QUARTER_OF_YEAR properly. + */ + ZoneId tz = ZoneId.of("Asia/Baghdad"); + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.QUARTER_OF_YEAR).timeZone(tz).build(); + assertThat(rounding.round(time("2006-12-31T13:21:44.308Z")), isDate(time("2006-09-30T20:00:00Z"), tz)); + } + + public void testPrepareLongRangeRoundsToMidnight() { + ZoneId tz = ZoneId.of("America/New_York"); + long min = time("01980-01-01T00:00:00Z"); + long max = time("10000-01-01T00:00:00Z"); + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.QUARTER_OF_YEAR).timeZone(tz).build(); + assertThat(rounding.round(time("2006-12-31T13:21:44.308Z")), isDate(time("2006-10-01T04:00:00Z"), tz)); + assertThat(rounding.round(time("9000-12-31T13:21:44.308Z")), isDate(time("9000-10-01T04:00:00Z"), tz)); + + Rounding.Prepared prepared = rounding.prepare(min, max); + assertThat(prepared.round(time("2006-12-31T13:21:44.308Z")), isDate(time("2006-10-01T04:00:00Z"), tz)); + assertThat(prepared.round(time("9000-12-31T13:21:44.308Z")), isDate(time("9000-10-01T04:00:00Z"), tz)); + } + + public void testPrepareLongRangeRoundsNotToMidnight() { + ZoneId tz = ZoneId.of("Australia/Lord_Howe"); + long min = time("01980-01-01T00:00:00Z"); + long max = time("10000-01-01T00:00:00Z"); + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build(); + assertThat(rounding.round(time("2018-03-31T15:25:15.148Z")), isDate(time("2018-03-31T14:00:00Z"), tz)); + assertThat(rounding.round(time("9000-03-31T15:25:15.148Z")), isDate(time("9000-03-31T15:00:00Z"), tz)); + + Rounding.Prepared prepared = rounding.prepare(min, max); + assertThat(prepared.round(time("2018-03-31T15:25:15.148Z")), isDate(time("2018-03-31T14:00:00Z"), tz)); + assertThat(prepared.round(time("9000-03-31T15:25:15.148Z")), isDate(time("9000-03-31T15:00:00Z"), tz)); + } + private void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes, ZoneId tz) { assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz); @@ -718,9 +793,9 @@ private void assertInterval(long rounded, long nextRoundingValue, Rounding round * @param rounding the rounding instance */ private void assertInterval(long rounded, long unrounded, long nextRoundingValue, Rounding rounding, ZoneId tz) { - assertThat("rounding should be idempotent ", rounding.round(rounded), isDate(rounded, tz)); - assertThat("rounded value smaller or equal than unrounded" + rounding, rounded, lessThanOrEqualTo(unrounded)); - assertThat("values less than rounded should round further down" + rounding, rounding.round(rounded - 1), lessThan(rounded)); + assertThat("rounding should be idempotent", rounding.round(rounded), isDate(rounded, tz)); + assertThat("rounded value smaller or equal than unrounded", rounded, lessThanOrEqualTo(unrounded)); + assertThat("values less than rounded should round further down", rounding.round(rounded - 1), lessThan(rounded)); assertThat("nextRounding value should be a rounded date", rounding.round(nextRoundingValue), isDate(nextRoundingValue, tz)); assertThat("values above nextRounding should round down there", rounding.round(nextRoundingValue + 1), isDate(nextRoundingValue, tz)); @@ -762,6 +837,18 @@ private static boolean isTimeWithWellDefinedRounding(ZoneId tz, long t) { return true; } + private static long randomDate() { + return Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00 + } + + private static long[] randomDateBounds() { + long b1 = randomDate(); + long b2 = randomValueOtherThan(b1, RoundingTests::randomDate); + if (b1 < b2) { + return new long[] {b1, b2}; + } + return new long[] {b2, b1}; + } private static long dateBetween(long lower, long upper) { long dateBetween = randomLongBetween(lower, upper - 1); assert lower <= dateBetween && dateBetween < upper; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 7443a823c0fba..98a58adfed6f2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; @@ -49,8 +50,17 @@ public class DateHistogramAggregatorTests extends AggregatorTestCase { - private static final String DATE_FIELD = "date"; - private static final String INSTANT_FIELD = "instant"; + /** + * A date that is always "aggregable" because it has doc values but may or + * may not have a search index. If it doesn't then we can't use our fancy + * date rounding mechanism that needs to know the minimum and maximum dates + * it is going to round because it ready *that* out of the search index. + */ + private static final String AGGREGABLE_DATE = "aggregable_date"; + /** + * A date that is always "searchable" because it is indexed. + */ + private static final String SEARCHABLE_DATE = "searchable_date"; private static final List dataset = Arrays.asList( "2010-03-12T01:07:45", @@ -66,7 +76,7 @@ public class DateHistogramAggregatorTests extends AggregatorTestCase { public void testMatchNoDocsDeprecatedInterval() throws IOException { testBothCases(new MatchNoDocsQuery(), dataset, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); @@ -77,11 +87,11 @@ public void testMatchNoDocsDeprecatedInterval() throws IOException { public void testMatchNoDocs() throws IOException { testBothCases(new MatchNoDocsQuery(), dataset, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); testBothCases(new MatchNoDocsQuery(), dataset, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(DATE_FIELD), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); } @@ -90,21 +100,21 @@ public void testMatchAllDocsDeprecatedInterval() throws IOException { Query query = new MatchAllDocsQuery(); testSearchCase(query, dataset, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(6, histogram.getBuckets().size()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); }, false ); testSearchAndReduceCase(query, dataset, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(8, histogram.getBuckets().size()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); }, false ); testBothCases(query, dataset, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { assertEquals(6, histogram.getBuckets().size()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); @@ -121,33 +131,34 @@ public void testMatchAllDocs() throws IOException { foo.add(dataset.get(randomIntBetween(0, dataset.size()-1))); } testSearchAndReduceCase(query, foo, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(DATE_FIELD).order(BucketOrder.count(false)), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")) + .field(AGGREGABLE_DATE).order(BucketOrder.count(false)), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); testSearchCase(query, dataset, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); testSearchAndReduceCase(query, dataset, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); testBothCases(query, dataset, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); testSearchCase(query, dataset, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(DATE_FIELD), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); testSearchAndReduceCase(query, dataset, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(DATE_FIELD), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); testBothCases(query, dataset, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); } @@ -156,7 +167,7 @@ public void testNoDocsDeprecatedInterval() throws IOException { Query query = new MatchNoDocsQuery(); List dates = Collections.emptyList(); Consumer aggregation = - agg -> agg.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD); + agg -> agg.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE); testSearchCase(query, dates, aggregation, histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -173,7 +184,7 @@ public void testNoDocs() throws IOException { Query query = new MatchNoDocsQuery(); List dates = Collections.emptyList(); Consumer aggregation = agg -> - agg.calendarInterval(DateHistogramInterval.YEAR).field(DATE_FIELD); + agg.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE); testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); @@ -182,7 +193,7 @@ public void testNoDocs() throws IOException { ); aggregation = agg -> - agg.fixedInterval(new DateHistogramInterval("365d")).field(DATE_FIELD); + agg.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE); testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); @@ -214,8 +225,8 @@ public void testAggregateWrongField() throws IOException { } public void testIntervalYearDeprecated() throws IOException { - testBothCases(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + testBothCases(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), dataset, + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -237,8 +248,8 @@ public void testIntervalYearDeprecated() throws IOException { } public void testIntervalYear() throws IOException { - testBothCases(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(DATE_FIELD), + testBothCases(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), dataset, + aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -261,7 +272,7 @@ public void testIntervalYear() throws IOException { public void testIntervalMonthDeprecated() throws IOException { testBothCases(new MatchAllDocsQuery(), Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MONTH).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MONTH).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -285,7 +296,7 @@ public void testIntervalMonthDeprecated() throws IOException { public void testIntervalMonth() throws IOException { testBothCases(new MatchAllDocsQuery(), Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), - aggregation -> aggregation.calendarInterval(DateHistogramInterval.MONTH).field(DATE_FIELD), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.MONTH).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -316,7 +327,7 @@ public void testIntervalDayDeprecated() throws IOException { "2017-02-03", "2017-02-05" ), - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.DAY).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.DAY).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(4, buckets.size()); @@ -352,7 +363,7 @@ public void testIntervalDay() throws IOException { "2017-02-03", "2017-02-05" ), - aggregation -> aggregation.calendarInterval(DateHistogramInterval.DAY).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.DAY).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(4, buckets.size()); @@ -384,7 +395,7 @@ public void testIntervalDay() throws IOException { "2017-02-03", "2017-02-05" ), - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("24h")).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("24h")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(4, buckets.size()); @@ -422,7 +433,7 @@ public void testIntervalHourDeprecated() throws IOException { "2017-02-01T16:48:00.000Z", "2017-02-01T16:59:00.000Z" ), - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.HOUR).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.HOUR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(6, buckets.size()); @@ -469,7 +480,7 @@ public void testIntervalHour() throws IOException { "2017-02-01T16:48:00.000Z", "2017-02-01T16:59:00.000Z" ), - aggregation -> aggregation.calendarInterval(DateHistogramInterval.HOUR).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.HOUR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(6, buckets.size()); @@ -512,7 +523,7 @@ public void testIntervalHour() throws IOException { "2017-02-01T16:48:00.000Z", "2017-02-01T16:59:00.000Z" ), - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(6, buckets.size()); @@ -553,7 +564,7 @@ public void testIntervalMinuteDeprecated() throws IOException { "2017-02-01T09:16:04.000Z", "2017-02-01T09:16:42.000Z" ), - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MINUTE).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MINUTE).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -583,7 +594,7 @@ public void testIntervalMinute() throws IOException { "2017-02-01T09:16:04.000Z", "2017-02-01T09:16:42.000Z" ), - aggregation -> aggregation.calendarInterval(DateHistogramInterval.MINUTE).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.MINUTE).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -609,7 +620,7 @@ public void testIntervalMinute() throws IOException { "2017-02-01T09:16:04.000Z", "2017-02-01T09:16:42.000Z" ), - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60s")).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60s")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -639,7 +650,7 @@ public void testIntervalSecondDeprecated() throws IOException { "2017-02-01T00:00:37.210Z", "2017-02-01T00:00:37.380Z" ), - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.SECOND).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.SECOND).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -670,7 +681,7 @@ public void testIntervalSecond() throws IOException { "2017-02-01T00:00:37.210Z", "2017-02-01T00:00:37.380Z" ), - aggregation -> aggregation.calendarInterval(DateHistogramInterval.SECOND).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.SECOND).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -697,7 +708,7 @@ public void testIntervalSecond() throws IOException { "2017-02-01T00:00:37.210Z", "2017-02-01T00:00:37.380Z" ), - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("1000ms")).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("1000ms")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -727,7 +738,7 @@ public void testNanosIntervalSecond() throws IOException { "2017-02-01T00:00:37.210328172Z", "2017-02-01T00:00:37.380889483Z" ), - aggregation -> aggregation.calendarInterval(DateHistogramInterval.SECOND).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.calendarInterval(DateHistogramInterval.SECOND).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -754,7 +765,7 @@ public void testNanosIntervalSecond() throws IOException { "2017-02-01T00:00:37.210328172Z", "2017-02-01T00:00:37.380889483Z" ), - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("1000ms")).field(DATE_FIELD).minDocCount(1L), + aggregation -> aggregation.fixedInterval(new DateHistogramInterval("1000ms")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(3, buckets.size()); @@ -775,7 +786,7 @@ public void testNanosIntervalSecond() throws IOException { } public void testMinDocCountDeprecated() throws IOException { - Query query = LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2017-02-01T00:00:00.000Z"), asLong("2017-02-01T00:00:30.000Z")); + Query query = LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2017-02-01T00:00:00.000Z"), asLong("2017-02-01T00:00:30.000Z")); List timestamps = Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -786,7 +797,7 @@ public void testMinDocCountDeprecated() throws IOException { // 5 sec interval with minDocCount = 0 testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(4, buckets.size()); @@ -811,7 +822,7 @@ public void testMinDocCountDeprecated() throws IOException { // 5 sec interval with minDocCount = 3 testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(3L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(3L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(1, buckets.size()); @@ -825,7 +836,7 @@ public void testMinDocCountDeprecated() throws IOException { } public void testMinDocCount() throws IOException { - Query query = LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2017-02-01T00:00:00.000Z"), asLong("2017-02-01T00:00:30.000Z")); + Query query = LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2017-02-01T00:00:00.000Z"), asLong("2017-02-01T00:00:30.000Z")); List timestamps = Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -836,7 +847,7 @@ public void testMinDocCount() throws IOException { // 5 sec interval with minDocCount = 0 testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L), + aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(4, buckets.size()); @@ -861,7 +872,7 @@ public void testMinDocCount() throws IOException { // 5 sec interval with minDocCount = 3 testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(3L), + aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(3L), histogram -> { List buckets = histogram.getBuckets(); assertEquals(1, buckets.size()); @@ -882,25 +893,25 @@ public void testMaxBucket() throws IOException { ); expectThrows(TooManyBucketsException.class, () -> testSearchCase(query, timestamps, - aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD), + aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE), histogram -> {}, 2, false)); expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD), + aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE), histogram -> {}, 2, false)); expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L), + aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> {}, 100, false)); expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps, aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)) - .field(DATE_FIELD) + .field(AGGREGABLE_DATE) .subAggregation( AggregationBuilders.dateHistogram("1") .fixedInterval(DateHistogramInterval.seconds(5)) - .field(DATE_FIELD) + .field(AGGREGABLE_DATE) ), histogram -> {}, 5, false)); } @@ -914,25 +925,25 @@ public void testMaxBucketDeprecated() throws IOException { ); expectThrows(TooManyBucketsException.class, () -> testSearchCase(query, timestamps, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE), histogram -> {}, 2, false)); expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE), histogram -> {}, 2, false)); expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L), + aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> {}, 100, false)); expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)) - .field(DATE_FIELD) + .field(AGGREGABLE_DATE) .subAggregation( AggregationBuilders.dateHistogram("1") .dateHistogramInterval(DateHistogramInterval.seconds(5)) - .field(DATE_FIELD) + .field(AGGREGABLE_DATE) ), histogram -> {}, 5, false)); assertWarnings("[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future."); @@ -949,7 +960,7 @@ public void testFixedWithCalendar() throws IOException { "2017-02-03", "2017-02-05" ), - aggregation -> aggregation.fixedInterval(DateHistogramInterval.WEEK).field(DATE_FIELD), + aggregation -> aggregation.fixedInterval(DateHistogramInterval.WEEK).field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("failed to parse setting [date_histogram.fixedInterval] with value [1w] as a time value: " + @@ -967,7 +978,7 @@ public void testCalendarWithFixed() throws IOException { "2017-02-03", "2017-02-05" ), - aggregation -> aggregation.calendarInterval(new DateHistogramInterval("5d")).field(DATE_FIELD), + aggregation -> aggregation.calendarInterval(new DateHistogramInterval("5d")).field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("The supplied interval [5d] could not be parsed as a calendar interval.")); @@ -986,7 +997,7 @@ public void testCalendarAndThenFixed() throws IOException { ), aggregation -> aggregation.calendarInterval(DateHistogramInterval.DAY) .fixedInterval(new DateHistogramInterval("2d")) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [fixed_interval] with [calendar_interval] configuration option.")); @@ -1005,7 +1016,7 @@ public void testFixedAndThenCalendar() throws IOException { ), aggregation -> aggregation.fixedInterval(new DateHistogramInterval("2d")) .calendarInterval(DateHistogramInterval.DAY) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [calendar_interval] with [fixed_interval] configuration option.")); @@ -1024,7 +1035,7 @@ public void testNewThenLegacy() throws IOException { ), aggregation -> aggregation.fixedInterval(new DateHistogramInterval("2d")) .dateHistogramInterval(DateHistogramInterval.DAY) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [interval] with [fixed_interval] or [calendar_interval] configuration options.")); @@ -1041,7 +1052,7 @@ public void testNewThenLegacy() throws IOException { ), aggregation -> aggregation.calendarInterval(DateHistogramInterval.DAY) .dateHistogramInterval(DateHistogramInterval.DAY) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [interval] with [fixed_interval] or [calendar_interval] configuration options.")); @@ -1058,7 +1069,7 @@ public void testNewThenLegacy() throws IOException { ), aggregation -> aggregation.fixedInterval(new DateHistogramInterval("2d")) .interval(1000) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [interval] with [fixed_interval] or [calendar_interval] configuration options.")); @@ -1075,7 +1086,7 @@ public void testNewThenLegacy() throws IOException { ), aggregation -> aggregation.calendarInterval(DateHistogramInterval.DAY) .interval(1000) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [interval] with [fixed_interval] or [calendar_interval] configuration options.")); @@ -1094,7 +1105,7 @@ public void testLegacyThenNew() throws IOException { ), aggregation -> aggregation .dateHistogramInterval(DateHistogramInterval.DAY) .fixedInterval(new DateHistogramInterval("2d")) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [fixed_interval] with [interval] configuration option.")); @@ -1111,7 +1122,7 @@ public void testLegacyThenNew() throws IOException { ), aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.DAY) .calendarInterval(DateHistogramInterval.DAY) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [calendar_interval] with [interval] configuration option.")); @@ -1128,7 +1139,7 @@ public void testLegacyThenNew() throws IOException { ), aggregation -> aggregation.interval(1000) .fixedInterval(new DateHistogramInterval("2d")) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [fixed_interval] with [interval] configuration option.")); @@ -1145,7 +1156,7 @@ public void testLegacyThenNew() throws IOException { ), aggregation -> aggregation.interval(1000) .calendarInterval(DateHistogramInterval.DAY) - .field(DATE_FIELD), + .field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Cannot use [calendar_interval] with [interval] configuration option.")); @@ -1156,7 +1167,7 @@ public void testLegacyThenNew() throws IOException { public void testIllegalInterval() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchAllDocsQuery(), Collections.emptyList(), - aggregation -> aggregation.dateHistogramInterval(new DateHistogramInterval("foobar")).field(DATE_FIELD), + aggregation -> aggregation.dateHistogramInterval(new DateHistogramInterval("foobar")).field(AGGREGABLE_DATE), histogram -> {}, false )); assertThat(e.getMessage(), equalTo("Unable to parse interval [foobar]")); @@ -1210,13 +1221,17 @@ private void executeTestCase(boolean reduced, Consumer verify, int maxBucket, boolean useNanosecondResolution) throws IOException { + boolean aggregableDateIsSearchable = randomBoolean(); + + DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name"); + if (useNanosecondResolution) { + builder.withResolution(DateFieldMapper.Resolution.NANOSECONDS); + } + DateFieldMapper.DateFieldType fieldType = builder.fieldType(); + fieldType.setHasDocValues(true); + fieldType.setIndexOptions(aggregableDateIsSearchable ? IndexOptions.DOCS : IndexOptions.NONE); + try (Directory directory = newDirectory()) { - DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name"); - if (useNanosecondResolution) { - builder.withResolution(DateFieldMapper.Resolution.NANOSECONDS); - } - DateFieldMapper.DateFieldType fieldType = builder.fieldType(); - fieldType.setHasDocValues(true); try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); @@ -1226,8 +1241,11 @@ private void executeTestCase(boolean reduced, } long instant = asLong(date, fieldType); - document.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); - document.add(new LongPoint(INSTANT_FIELD, instant)); + document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant)); + if (aggregableDateIsSearchable) { + document.add(new LongPoint(AGGREGABLE_DATE, instant)); + } + document.add(new LongPoint(SEARCHABLE_DATE, instant)); indexWriter.addDocument(document); document.clear(); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java index acee17d3fdbe1..6f692a47a3a5a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java @@ -6,19 +6,29 @@ package org.elasticsearch.xpack.analytics.aggregations.support; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.Rounding.Prepared; import org.elasticsearch.index.fielddata.DocValueBits; import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import java.io.IOException; +import java.util.function.Function; public class HistogramValuesSource { public abstract static class Histogram extends org.elasticsearch.search.aggregations.support.ValuesSource { public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException; + @Override + public Function roundingPreparer(IndexReader reader) throws IOException { + throw new AggregationExecutionException("can't round a [histogram]"); + } + public static class Fielddata extends Histogram { protected final IndexHistogramFieldData indexFieldData; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSource.java index c1de13ffb44dd..98aca008392d6 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSource.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSource.java @@ -6,15 +6,20 @@ package org.elasticsearch.xpack.spatial.search.aggregations.support; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.Rounding.Prepared; import org.elasticsearch.index.fielddata.DocValueBits; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.xpack.spatial.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; import java.io.IOException; +import java.util.function.Function; public abstract class GeoShapeValuesSource extends ValuesSource { public static final GeoShapeValuesSource EMPTY = new GeoShapeValuesSource() { @@ -33,6 +38,11 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc public abstract MultiGeoShapeValues geoShapeValues(LeafReaderContext context); + @Override + public Function roundingPreparer(IndexReader reader) throws IOException { + throw new AggregationExecutionException("can't round a [geo_shape]"); + } + @Override public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { MultiGeoShapeValues values = geoShapeValues(context);