Add DNS recursor strategy option

.changelog/10611.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+config: add `dns_config.recursor_strategy` flag to control the order which DNS recursors are queried
+```

agent/config/builder.go

@@ -910,6 +910,7 @@ func (b *builder) Build() (rt RuntimeConfig, err error) {
 		DNSNodeTTL:            b.durationVal("dns_config.node_ttl", c.DNS.NodeTTL),
 		DNSOnlyPassing:        boolVal(c.DNS.OnlyPassing),
 		DNSPort:               dnsPort,
+		DNSRecursorStrategy:   b.dnsRecursorStrategyVal(stringVal(c.DNS.RecursorStrategy)),
 		DNSRecursorTimeout:    b.durationVal("recursor_timeout", c.DNS.RecursorTimeout),
 		DNSRecursors:          dnsRecursors,
 		DNSServiceTTL:         dnsServiceTTL,
@@ -1747,6 +1748,20 @@ func (b *builder) meshGatewayConfVal(mgConf *MeshGatewayConfig) structs.MeshGate
 	return cfg
 }
 
+func (b *builder) dnsRecursorStrategyVal(v string) dns.RecursorStrategy {
+	var out dns.RecursorStrategy
+
+	switch dns.RecursorStrategy(v) {
+	case dns.RecursorStrategyRandom:
+		out = dns.RecursorStrategyRandom
+	case dns.RecursorStrategySequential, "":
+		out = dns.RecursorStrategySequential
+	default:
+		b.err = multierror.Append(b.err, fmt.Errorf("dns_config.recursor_strategy: invalid strategy: %q", v))
+	}
+	return out
+}
+
 func (b *builder) exposeConfVal(v *ExposeConfig) structs.ExposeConfig {
 	var out structs.ExposeConfig
 	if v == nil {

agent/config/config.go

@@ -634,6 +634,7 @@ type DNS struct {
 	MaxStale           *string           `mapstructure:"max_stale"`
 	NodeTTL            *string           `mapstructure:"node_ttl"`
 	OnlyPassing        *bool             `mapstructure:"only_passing"`
+	RecursorStrategy   *string           `mapstructure:"recursor_strategy"`
 	RecursorTimeout    *string           `mapstructure:"recursor_timeout"`
 	ServiceTTL         map[string]string `mapstructure:"service_ttl"`
 	UDPAnswerLimit     *int              `mapstructure:"udp_answer_limit"`

agent/config/runtime.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/hashicorp/consul/agent/cache"
 	"github.com/hashicorp/consul/agent/consul"
+	"github.com/hashicorp/consul/agent/dns"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/api"
@@ -270,6 +271,15 @@ type RuntimeConfig struct {
 	// hcl: dns_config { only_passing = (true|false) }
 	DNSOnlyPassing bool
 
+	// DNSRecursorStrategy controls the order in which DNS recursors are queried.
+	// 'sequential' queries recursors in the order they are listed under `recursors`.
+	// 'random' causes random selection of recursors which has the effect of
+	// spreading the query load among all listed servers, rather than having
+	// client agents try the first server in the list every time.
+	//
+	// hcl: dns_config { recursor_strategy = "(random|sequential)" }
+	DNSRecursorStrategy dns.RecursorStrategy
+
 	// DNSRecursorTimeout specifies the timeout in seconds
 	// for Consul's internal dns client used for recursion.
 	// This value is used for the connection, read and write timeout.

agent/config/runtime_test.go

@@ -5430,6 +5430,7 @@ func TestLoad_FullConfig(t *testing.T) {
 		DNSNodeTTL:                             7084 * time.Second,
 		DNSOnlyPassing:                         true,
 		DNSPort:                                7001,
+		DNSRecursorStrategy:                    "sequential",
 		DNSRecursorTimeout:                     4427 * time.Second,
 		DNSRecursors:                           []string{"63.38.39.58", "92.49.18.18"},
 		DNSSOA:                                 RuntimeSOAConfig{Refresh: 3600, Retry: 600, Expire: 86400, Minttl: 0},

agent/dns.go

@@ -69,22 +69,23 @@ type dnsSOAConfig struct {
 }
 
 type dnsConfig struct {
-	AllowStale      bool
-	Datacenter      string
-	EnableTruncate  bool
-	MaxStale        time.Duration
-	UseCache        bool
-	CacheMaxAge     time.Duration
-	NodeName        string
-	NodeTTL         time.Duration
-	OnlyPassing     bool
-	RecursorTimeout time.Duration
-	Recursors       []string
-	SegmentName     string
-	UDPAnswerLimit  int
-	ARecordLimit    int
-	NodeMetaTXT     bool
-	SOAConfig       dnsSOAConfig
+	AllowStale       bool
+	Datacenter       string
+	EnableTruncate   bool
+	MaxStale         time.Duration
+	UseCache         bool
+	CacheMaxAge      time.Duration
+	NodeName         string
+	NodeTTL          time.Duration
+	OnlyPassing      bool
+	RecursorStrategy agentdns.RecursorStrategy
+	RecursorTimeout  time.Duration
+	Recursors        []string
+	SegmentName      string
+	UDPAnswerLimit   int
+	ARecordLimit     int
+	NodeMetaTXT      bool
+	SOAConfig        dnsSOAConfig
 	// TTLRadix sets service TTLs by prefix, eg: "database-*"
 	TTLRadix *radix.Tree
 	// TTLStict sets TTLs to service by full name match. It Has higher priority than TTLRadix
@@ -154,6 +155,7 @@ func GetDNSConfig(conf *config.RuntimeConfig) (*dnsConfig, error) {
 		NodeName:           conf.NodeName,
 		NodeTTL:            conf.DNSNodeTTL,
 		OnlyPassing:        conf.DNSOnlyPassing,
+		RecursorStrategy:   conf.DNSRecursorStrategy,
 		RecursorTimeout:    conf.DNSRecursorTimeout,
 		SegmentName:        conf.SegmentName,
 		UDPAnswerLimit:     conf.DNSUDPAnswerLimit,
@@ -1844,7 +1846,8 @@ func (d *DNSServer) handleRecurse(resp dns.ResponseWriter, req *dns.Msg) {
 	var r *dns.Msg
 	var rtt time.Duration
 	var err error
-	for _, recursor := range cfg.Recursors {
+	for _, idx := range cfg.RecursorStrategy.Indexes(len(cfg.Recursors)) {
+		recursor := cfg.Recursors[idx]
 		r, rtt, err = c.Exchange(req, recursor)
 		// Check if the response is valid and has the desired Response code
 		if r != nil && (r.Rcode != dns.RcodeSuccess && r.Rcode != dns.RcodeNameError) {
@@ -1928,7 +1931,8 @@ func (d *DNSServer) resolveCNAME(cfg *dnsConfig, name string, maxRecursionLevel
 	var r *dns.Msg
 	var rtt time.Duration
 	var err error
-	for _, recursor := range cfg.Recursors {
+	for _, idx := range cfg.RecursorStrategy.Indexes(len(cfg.Recursors)) {
+		recursor := cfg.Recursors[idx]
 		r, rtt, err = c.Exchange(m, recursor)
 		if err == nil {
 			d.logger.Debug("cname recurse RTT for name",

agent/dns/dns.go

@@ -1,10 +1,34 @@
 package dns
 
-import "regexp"
+import (
+	"math/rand"
+	"regexp"
+)
 
 // MaxLabelLength is the maximum length for a name that can be used in DNS.
 const MaxLabelLength = 63
 
 // InvalidNameRe is a regex that matches characters which can not be included in
 // a DNS name.
 var InvalidNameRe = regexp.MustCompile(`[^A-Za-z0-9\\-]+`)
+
+type RecursorStrategy string
+
+const (
+	RecursorStrategySequential RecursorStrategy = "sequential"
+	RecursorStrategyRandom     RecursorStrategy = "random"
+)
+
+func (s RecursorStrategy) Indexes(max int) []int {
+	switch s {
+	case RecursorStrategyRandom:
+		return rand.Perm(max)
+	default:
+		idxs := make([]int, max)
+		for i := range idxs {
+			idxs[i] = i
+		}
+		return idxs
+
+	}
+}

agent/dns_test.go

@@ -7471,6 +7471,7 @@ func TestDNS_ConfigReload(t *testing.T) {
 			}
 			enable_truncate = false
 			only_passing = false
+			recursor_strategy = "sequential"
 			recursor_timeout = "15s"
 			disable_compression = false
 			a_record_limit = 1
@@ -7489,6 +7490,7 @@ func TestDNS_ConfigReload(t *testing.T) {
 	for _, s := range a.dnsServers {
 		cfg := s.config.Load().(*dnsConfig)
 		require.Equal(t, []string{"8.8.8.8:53"}, cfg.Recursors)
+		require.Equal(t, "sequential", cfg.RecursorStrategy)
 		require.False(t, cfg.AllowStale)
 		require.Equal(t, 20*time.Second, cfg.MaxStale)
 		require.Equal(t, 10*time.Second, cfg.NodeTTL)
@@ -7519,6 +7521,7 @@ func TestDNS_ConfigReload(t *testing.T) {
 	}
 	newCfg.DNSEnableTruncate = true
 	newCfg.DNSOnlyPassing = true
+	newCfg.DNSRecursorStrategy = "random"
 	newCfg.DNSRecursorTimeout = 16 * time.Second
 	newCfg.DNSDisableCompression = true
 	newCfg.DNSARecordLimit = 2
@@ -7534,6 +7537,7 @@ func TestDNS_ConfigReload(t *testing.T) {
 	for _, s := range a.dnsServers {
 		cfg := s.config.Load().(*dnsConfig)
 		require.Equal(t, []string{"1.1.1.1:53"}, cfg.Recursors)
+		require.Equal(t, "random", cfg.RecursorStrategy)
 		require.True(t, cfg.AllowStale)
 		require.Equal(t, 21*time.Second, cfg.MaxStale)
 		require.Equal(t, 11*time.Second, cfg.NodeTTL)

website/content/docs/agent/options.mdx

@@ -1357,6 +1357,11 @@ bind_addr = "{{ GetPrivateInterfaces | include \"network\" \"10.0.0.0/8\" | attr
     then all services on that node will be excluded because they are also considered
     critical.
 
+  - `recursor_strategy` - If set to `sequential`, Consul will query recursors in the
+    order listed in the [`recursors`](#recursors) option. If set to `random`,
+    Consul will query an upstream DNS resolvers in a random order. Defaults to
+    `sequential`.
+
   - `recursor_timeout` - Timeout used by Consul when
     recursively querying an upstream DNS server. See [`recursors`](#recursors) for more details. Default is 2s. This is available in Consul 0.7 and later.