Add jitter for the subsequent DNS queries (#2170)

Motivation:

Current DNS implementation of the `ServiceDiscoverer` sends new queries
after each TTL expires. This may lead to spikes in load on DNS servers
if many clients align their requests based on the returned TTL value.

Modifications:

- Add jitter for the subsequent DNS queries;
- Allow users to customize jitter value;

Result:

Subsequent DNS queries are spread out in time using jitter.

servicetalk-dns-discovery-netty/build.gradle

@@ -28,6 +28,7 @@ dependencies {
   implementation project(":servicetalk-concurrent-api-internal")
   implementation project(":servicetalk-transport-netty")
   implementation project(":servicetalk-transport-netty-internal")
+  implementation project(":servicetalk-utils-internal")
   implementation "com.google.code.findbugs:jsr305:$jsr305Version"
   implementation "io.netty:netty-resolver-dns"
   implementation "org.slf4j:slf4j-api:$slf4jVersion"

servicetalk-dns-discovery-netty/src/main/java/io/servicetalk/dns/discovery/netty/DefaultDnsClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2018, 2021 Apple Inc. and the ServiceTalk project authors
+ * Copyright © 2018, 2021-2022 Apple Inc. and the ServiceTalk project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -64,6 +64,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.RandomAccess;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.IntFunction;
 import javax.annotation.Nullable;
 
@@ -117,9 +118,11 @@ final class DefaultDnsClient implements DnsClient {
     private final int srvConcurrency;
     private final boolean srvFilterDuplicateEvents;
     private final boolean inactiveEventsOnError;
+    private final long ttlJitterNanos;
     private boolean closed;
 
-    DefaultDnsClient(final IoExecutor ioExecutor, final int minTTL, int srvConcurrency, boolean inactiveEventsOnError,
+    DefaultDnsClient(final IoExecutor ioExecutor, final int minTTL, final long ttlJitterNanos,
+                     final int srvConcurrency, final boolean inactiveEventsOnError,
                      final boolean completeOncePreferredResolved, final boolean srvFilterDuplicateEvents,
                      Duration srvHostNameRepeatInitialDelay, Duration srvHostNameRepeatJitter,
                      @Nullable Integer maxUdpPayloadSize, @Nullable final Integer ndots,
@@ -141,6 +144,7 @@ final class DefaultDnsClient implements DnsClient {
                 srvHostNameRepeatInitialDelay, srvHostNameRepeatJitter, nettyIoExecutor);
         this.ttlCache = new MinTtlCache(new DefaultDnsCache(minTTL, Integer.MAX_VALUE, minTTL), minTTL,
                 nettyIoExecutor);
+        this.ttlJitterNanos = ttlJitterNanos;
         this.observer = observer;
         this.missingRecordStatus = missingRecordStatus;
         asyncCloseable = toAsyncCloseable(graceful -> {
@@ -629,12 +633,15 @@ private void cancelAndTerminate0(Throwable cause) {
             private void scheduleQuery0(final long nanos) {
                 assertInEventloop();
 
-                LOGGER.trace("DnsClient {}, scheduling DNS query for {} after {} nanos.",
-                        DefaultDnsClient.this, AbstractDnsPublisher.this, nanos);
+                final long delay = ThreadLocalRandom.current()
+                        .nextLong(nanos, addWithOverflowProtection(nanos, ttlJitterNanos));
+                LOGGER.debug("DnsClient {}, scheduling DNS query for {} after {}ms, original TTL: {}ms.",
+                        DefaultDnsClient.this, AbstractDnsPublisher.this, NANOSECONDS.toMillis(delay),
+                        NANOSECONDS.toMillis(nanos));
 
                 // This value is coming from DNS TTL for which the unit is seconds and the minimum value we accept
                 // in the builder is 1 second.
-                cancellableForQuery = nettyIoExecutor.schedule(this::doQuery0, nanos, NANOSECONDS);
+                cancellableForQuery = nettyIoExecutor.schedule(this::doQuery0, delay, NANOSECONDS);
             }
 
             private void handleResolveDone0(final Future<DnsAnswer<T>> addressFuture,

servicetalk-dns-discovery-netty/src/main/java/io/servicetalk/dns/discovery/netty/DefaultDnsServiceDiscovererBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2018, 2021 Apple Inc. and the ServiceTalk project authors
+ * Copyright © 2018, 2021-2022 Apple Inc. and the ServiceTalk project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,6 +29,7 @@
 import static io.servicetalk.dns.discovery.netty.DnsClients.asHostAndPortDiscoverer;
 import static io.servicetalk.dns.discovery.netty.DnsClients.asSrvDiscoverer;
 import static io.servicetalk.transport.netty.internal.GlobalExecutionContext.globalExecutionContext;
+import static io.servicetalk.utils.internal.DurationUtils.ensurePositive;
 import static java.time.Duration.ofSeconds;
 import static java.util.Objects.requireNonNull;
 
@@ -52,6 +53,7 @@ public final class DefaultDnsServiceDiscovererBuilder {
     @Nullable
     private Duration queryTimeout;
     private int minTTLSeconds = 10;
+    private Duration ttlJitter = ofSeconds(4);
     private int srvConcurrency = 2048;
     private boolean inactiveEventsOnError;
     private boolean completeOncePreferredResolved = true;
@@ -78,6 +80,21 @@ public DefaultDnsServiceDiscovererBuilder minTTL(final int minTTLSeconds) {
         return this;
     }
 
+    /**
+     * The jitter to apply to schedule the next query after TTL.
+     * <p>
+     * The jitter value will be added on top of the TTL value returned from the DNS server to help spread out
+     * subsequent DNS queries.
+     *
+     * @param ttlJitter The jitter to apply to schedule the next query after TTL.
+     * @return {@code this}.
+     */
+    public DefaultDnsServiceDiscovererBuilder ttlJitter(final Duration ttlJitter) {
+        ensurePositive(ttlJitter, "jitter");
+        this.ttlJitter = ttlJitter;
+        return this;
+    }
+
     /**
      * Set the {@link DnsServerAddressStreamProvider} which determines which DNS server should be used per query.
      *
@@ -288,7 +305,8 @@ private static DnsClientFilterFactory appendFilter(@Nullable final DnsClientFilt
      */
     DnsClient build() {
         final DnsClient rawClient = new DefaultDnsClient(
-                ioExecutor == null ? globalExecutionContext().ioExecutor() : ioExecutor, minTTLSeconds, srvConcurrency,
+                ioExecutor == null ? globalExecutionContext().ioExecutor() : ioExecutor, minTTLSeconds,
+                ttlJitter.toNanos(), srvConcurrency,
                 inactiveEventsOnError, completeOncePreferredResolved, srvFilterDuplicateEvents,
                 srvHostNameRepeatInitialDelay, srvHostNameRepeatJitter, maxUdpPayloadSize, ndots, optResourceEnabled,
                 queryTimeout, dnsResolverAddressTypes, dnsServerAddressStreamProvider, observer, missingRecordStatus);