From 2199e02665994b7f078b8fac244e171277817cde Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Fri, 13 Jun 2014 11:38:34 -0400 Subject: [PATCH 1/8] first pass at DNS discovery * Doesn't handle failures gracefully * Requires that you're in AWS * Doesn't respect TTL's --- dns_discover.go | 91 ++++++++++++++++++++++++++++++++++++++++++++ dns_discover_test.go | 57 +++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 dns_discover.go create mode 100644 dns_discover_test.go diff --git a/dns_discover.go b/dns_discover.go new file mode 100644 index 0000000..1fe8a58 --- /dev/null +++ b/dns_discover.go @@ -0,0 +1,91 @@ +package fargo + +// MIT Licensed (see README.md) - Copyright (c) 2013 Hudl <@Hudl> + +import ( + "fmt" + "github.com/franela/goreq" + "github.com/miekg/dns" + "time" +) + +const azURL = "http://169.254.169.254/latest/meta-data/placement/availability-zone" + +var ErrNotInAWS = fmt.Errorf("Not in AWS") + +func discoverDNS(domain string) (servers []string, ttl time.Duration, err error) { + // all DNS queries must use the FQDN + + z, _ := availabilityZone() + domain = "txt." + z + "." + dns.Fqdn(domain) + if _, ok := dns.IsDomainName(domain); !ok { + err = fmt.Errorf("invalid domain name: '%s' is not a domain name", domain) + return + } + availabilityZoneRecords, err := findTXT(domain) + if err != nil { + return + } + + for _, az := range availabilityZoneRecords { + s, er := findTXT("txt." + dns.Fqdn(az)) + if er != nil { + continue + } + servers = append(servers, s...) + } + return +} + +func findTXT(fqdn string) ([]string, error) { + query := new(dns.Msg) + query.SetQuestion(fqdn, dns.TypeTXT) + response, err := dns.Exchange(query, dnsServerAddr) + if err != nil { + log.Error("Failure resolving name %s err=%s", fqdn, err.Error()) + return nil, err + } + if len(response.Answer) < 1 { + err := fmt.Errorf("no Eureka discovery TXT record returned for name=%s", fqdn) + log.Error("no answer for name=%s err=%s", fqdn, err.Error()) + return nil, err + } + if response.Answer[0].Header().Rrtype != dns.TypeTXT { + err := fmt.Errorf("did not receive TXT record back from query specifying TXT record. This should never happen.") + log.Error("Failure resolving name %s err=%s", fqdn, err.Error()) + return nil, err + } + txt := response.Answer[0].(*dns.TXT) + + return txt.Txt, nil +} + +var dnsServerAddr string + +func init() { + // this is all a very verbose way + config, _ := dns.ClientConfigFromFile("/etc/resolv.conf") + dnsServerAddr = config.Servers[0] + ":" + config.Port + //findTXT("txt.us-east-1.app.hudl.com.") + //findTXT("txt.us-east-1.discoverytest.netflix.net.") +} + +// defaults to us-east-1 if there's a problem +func availabilityZone() (string, error) { + response, err := goreq.Request{Uri: azURL}.Do() + if err != nil { + return "us-east-1", err + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + body, _ := response.Body.ToString() + return "us-east-1", fmt.Errorf("bad response code: code %d does not indicate successful request, body=%s", + response.StatusCode, + body, + ) + } + zone, err := response.Body.ToString() + if err != nil { + return "us-east-1", err + } + return zone[:len(zone)-1], nil +} diff --git a/dns_discover_test.go b/dns_discover_test.go new file mode 100644 index 0000000..5652a5b --- /dev/null +++ b/dns_discover_test.go @@ -0,0 +1,57 @@ +package fargo + +// MIT Licensed (see README.md) - Copyright (c) 2013 Hudl <@Hudl> + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestGetNXDomain(t *testing.T) { + Convey("Given nonexistent domain nxd.local.", t, func() { + resp, err := findTXT("nxd.local.") + So(err, ShouldNotBeNil) + So(len(resp), ShouldEqual, 0) + }) +} + +func TestGetNetflixTestDomain(t *testing.T) { + Convey("Given domain txt.us-east-1.discoverytest.netflix.net.", t, func() { + // TODO: use a mock DNS server to eliminate dependency on netflix + // keeping their discoverytest domain up + resp, err := findTXT("txt.us-east-1.discoverytest.netflix.net.") + So(err, ShouldBeNil) + So(len(resp), ShouldEqual, 3) + Convey("And the contents are zone records", func() { + expected := map[string]bool{ + "us-east-1c.us-east-1.discoverytest.netflix.net": true, + "us-east-1d.us-east-1.discoverytest.netflix.net": true, + "us-east-1e.us-east-1.discoverytest.netflix.net": true, + } + for _, item := range resp { + _, ok := expected[item] + So(ok, ShouldEqual, true) + } + Convey("And the zone records contain instances", func() { + for _, record := range resp { + servers, err := findTXT("txt." + record + ".") + So(err, ShouldBeNil) + So(len(servers) >= 1, ShouldEqual, true) + // servers should be EC2 DNS names + So(servers[0][0:4], ShouldEqual, "ec2-") + } + }) + }) + }) + Convey("Autodiscover discoverytest.netflix.net.", t, func() { + servers, ttl, err := discoverDNS("discoverytest.netflix.net") + _ = ttl + So(err, ShouldBeNil) + So(len(servers), ShouldEqual, 4) + Convey("Servers discovered should all be EC2 DNS names", func() { + for _, s := range servers { + So(s[0:4], ShouldEqual, "ec2-") + } + }) + }) +} From 794a62e3c385fa4fd7e0b915f45bc30f601693de Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Fri, 13 Jun 2014 14:00:33 -0400 Subject: [PATCH 2/8] correct comment --- dns_discover.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dns_discover.go b/dns_discover.go index 1fe8a58..5a25838 100644 --- a/dns_discover.go +++ b/dns_discover.go @@ -63,11 +63,9 @@ func findTXT(fqdn string) ([]string, error) { var dnsServerAddr string func init() { - // this is all a very verbose way + // Find a DNS server using the OS resolv.conf config, _ := dns.ClientConfigFromFile("/etc/resolv.conf") dnsServerAddr = config.Servers[0] + ":" + config.Port - //findTXT("txt.us-east-1.app.hudl.com.") - //findTXT("txt.us-east-1.discoverytest.netflix.net.") } // defaults to us-east-1 if there's a problem From 2dd59e003378f31c85702266395e159b179202b8 Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Mon, 16 Jun 2014 10:17:53 -0500 Subject: [PATCH 3/8] return the TTL of the region record for expiry purposes --- dns_discover.go | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/dns_discover.go b/dns_discover.go index 5a25838..72e651f 100644 --- a/dns_discover.go +++ b/dns_discover.go @@ -14,21 +14,21 @@ const azURL = "http://169.254.169.254/latest/meta-data/placement/availability-zo var ErrNotInAWS = fmt.Errorf("Not in AWS") func discoverDNS(domain string) (servers []string, ttl time.Duration, err error) { - // all DNS queries must use the FQDN + r, _ := region() - z, _ := availabilityZone() - domain = "txt." + z + "." + dns.Fqdn(domain) + // all DNS queries must use the FQDN + domain = "txt." + r + "." + dns.Fqdn(domain) if _, ok := dns.IsDomainName(domain); !ok { err = fmt.Errorf("invalid domain name: '%s' is not a domain name", domain) return } - availabilityZoneRecords, err := findTXT(domain) + regionRecords, ttl, err := findTXT(domain) if err != nil { return } - for _, az := range availabilityZoneRecords { - s, er := findTXT("txt." + dns.Fqdn(az)) + for _, az := range regionRecords { + s, _, er := findTXT("txt." + dns.Fqdn(az)) if er != nil { continue } @@ -37,27 +37,29 @@ func discoverDNS(domain string) (servers []string, ttl time.Duration, err error) return } -func findTXT(fqdn string) ([]string, error) { +func findTXT(fqdn string) ([]string, time.Duration, error) { + defaultTTL := 120 * time.Second query := new(dns.Msg) query.SetQuestion(fqdn, dns.TypeTXT) response, err := dns.Exchange(query, dnsServerAddr) if err != nil { log.Error("Failure resolving name %s err=%s", fqdn, err.Error()) - return nil, err + return nil, defaultTTL, err } if len(response.Answer) < 1 { err := fmt.Errorf("no Eureka discovery TXT record returned for name=%s", fqdn) log.Error("no answer for name=%s err=%s", fqdn, err.Error()) - return nil, err + return nil, defaultTTL, err } if response.Answer[0].Header().Rrtype != dns.TypeTXT { err := fmt.Errorf("did not receive TXT record back from query specifying TXT record. This should never happen.") log.Error("Failure resolving name %s err=%s", fqdn, err.Error()) - return nil, err + return nil, defaultTTL, err } txt := response.Answer[0].(*dns.TXT) + ttl := response.Answer[0].Header().Ttl - return txt.Txt, nil + return txt.Txt, time.Duration(ttl) * time.Second, nil } var dnsServerAddr string @@ -68,22 +70,31 @@ func init() { dnsServerAddr = config.Servers[0] + ":" + config.Port } +func region() (string, error) { + zone, err := availabilityZone() + if err != nil { + log.Error("Could not retrieve availability zone err=%s", err.Error()) + return "us-east-1", err + } + return zone[:len(zone)-1], nil +} + // defaults to us-east-1 if there's a problem func availabilityZone() (string, error) { response, err := goreq.Request{Uri: azURL}.Do() if err != nil { - return "us-east-1", err + return "", err } if response.StatusCode < 200 || response.StatusCode >= 300 { body, _ := response.Body.ToString() - return "us-east-1", fmt.Errorf("bad response code: code %d does not indicate successful request, body=%s", + return "", fmt.Errorf("bad response code: code %d does not indicate successful request, body=%s", response.StatusCode, body, ) } zone, err := response.Body.ToString() if err != nil { - return "us-east-1", err + return "", err } return zone[:len(zone)-1], nil } From 8dde6baac417417c665c02d63b45e9d90d62eef7 Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Tue, 17 Jun 2014 14:21:23 -0500 Subject: [PATCH 4/8] add timer for DNS TTLs --- config.go | 1 + connection.go | 18 ++++++++++++++---- dns_discover.go | 3 +++ dns_discover_test.go | 10 ++++++---- struct.go | 3 +++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/config.go b/config.go index 948bd1a..c3f3cba 100644 --- a/config.go +++ b/config.go @@ -30,6 +30,7 @@ type eureka struct { InTheCloud bool // default false ConnectTimeoutSeconds int // default 10s UseDNSForServiceUrls bool // default false + DNSDiscoveryZone string // default "" ServerDNSName string // default "" ServiceUrls []string // default [] ServerPort int // default 7001 diff --git a/connection.go b/connection.go index 65487a0..aba47fb 100644 --- a/connection.go +++ b/connection.go @@ -15,6 +15,15 @@ func init() { // balancing scheme. // TODO: Make this not just pick a random one. func (e *EurekaConnection) SelectServiceURL() string { + if len(e.discoveryTtl) > 0 { + <-e.discoveryTtl + servers, ttl, err := discoverDNS(e.DiscoveryZone) + if err != nil { + return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] + } + e.discoveryTtl = time.After(ttl) + e.ServiceUrls = servers + } return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] } @@ -32,10 +41,6 @@ func NewConnFromConfigFile(location string) (c EurekaConnection, err error) { // NewConnFromConfig will, given a Config struct, return a connection based on // those options func NewConnFromConfig(conf Config) (c EurekaConnection) { - if conf.Eureka.UseDNSForServiceUrls { - //TODO: Read service urls from DNS TXT records - log.Critical("ERROR: UseDNSForServiceUrls option unsupported.") - } c.ServiceUrls = conf.Eureka.ServiceUrls if len(c.ServiceUrls) == 0 && len(conf.Eureka.ServerDNSName) > 0 { c.ServiceUrls = []string{conf.Eureka.ServerDNSName} @@ -43,6 +48,11 @@ func NewConnFromConfig(conf Config) (c EurekaConnection) { c.Timeout = time.Duration(conf.Eureka.ConnectTimeoutSeconds) * time.Second c.PollInterval = time.Duration(conf.Eureka.PollIntervalSeconds) * time.Second c.PreferSameZone = conf.Eureka.PreferSameZone + if conf.Eureka.UseDNSForServiceUrls { + log.Warning("UseDNSForServiceUrls is an experimental option") + c.DNSDiscovery = true + c.DiscoveryZone = conf.Eureka.DNSDiscoveryZone + } return c } diff --git a/dns_discover.go b/dns_discover.go index 72e651f..4e9c9f7 100644 --- a/dns_discover.go +++ b/dns_discover.go @@ -58,6 +58,9 @@ func findTXT(fqdn string) ([]string, time.Duration, error) { } txt := response.Answer[0].(*dns.TXT) ttl := response.Answer[0].Header().Ttl + if ttl < 60 { + ttl = 60 + } return txt.Txt, time.Duration(ttl) * time.Second, nil } diff --git a/dns_discover_test.go b/dns_discover_test.go index 5652a5b..f1b6414 100644 --- a/dns_discover_test.go +++ b/dns_discover_test.go @@ -5,11 +5,12 @@ package fargo import ( . "github.com/smartystreets/goconvey/convey" "testing" + "time" ) func TestGetNXDomain(t *testing.T) { Convey("Given nonexistent domain nxd.local.", t, func() { - resp, err := findTXT("nxd.local.") + resp, _, err := findTXT("nxd.local.") So(err, ShouldNotBeNil) So(len(resp), ShouldEqual, 0) }) @@ -19,8 +20,9 @@ func TestGetNetflixTestDomain(t *testing.T) { Convey("Given domain txt.us-east-1.discoverytest.netflix.net.", t, func() { // TODO: use a mock DNS server to eliminate dependency on netflix // keeping their discoverytest domain up - resp, err := findTXT("txt.us-east-1.discoverytest.netflix.net.") + resp, ttl, err := findTXT("txt.us-east-1.discoverytest.netflix.net.") So(err, ShouldBeNil) + So(ttl, ShouldEqual, 60*time.Second) So(len(resp), ShouldEqual, 3) Convey("And the contents are zone records", func() { expected := map[string]bool{ @@ -34,7 +36,7 @@ func TestGetNetflixTestDomain(t *testing.T) { } Convey("And the zone records contain instances", func() { for _, record := range resp { - servers, err := findTXT("txt." + record + ".") + servers, _, err := findTXT("txt." + record + ".") So(err, ShouldBeNil) So(len(servers) >= 1, ShouldEqual, true) // servers should be EC2 DNS names @@ -45,7 +47,7 @@ func TestGetNetflixTestDomain(t *testing.T) { }) Convey("Autodiscover discoverytest.netflix.net.", t, func() { servers, ttl, err := discoverDNS("discoverytest.netflix.net") - _ = ttl + So(ttl, ShouldEqual, 60*time.Second) So(err, ShouldBeNil) So(len(servers), ShouldEqual, 4) Convey("Servers discovered should all be EC2 DNS names", func() { diff --git a/struct.go b/struct.go index 6cd4747..0e8b511 100644 --- a/struct.go +++ b/struct.go @@ -19,6 +19,9 @@ type EurekaConnection struct { PollInterval time.Duration PreferSameZone bool Retries int + DNSDiscovery bool + DiscoveryZone string + discoveryTtl <-chan time.Time } // GetAppsResponse lets us deserialize the eureka/v2/apps response XML From 0b774cc830d8692e99e3a16a8ee8f57cf0fd0b12 Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Tue, 17 Jun 2014 14:22:40 -0500 Subject: [PATCH 5/8] only check DNS timeout if DNS discovery is enabled --- connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connection.go b/connection.go index aba47fb..495016c 100644 --- a/connection.go +++ b/connection.go @@ -15,7 +15,7 @@ func init() { // balancing scheme. // TODO: Make this not just pick a random one. func (e *EurekaConnection) SelectServiceURL() string { - if len(e.discoveryTtl) > 0 { + if e.DNSDiscovery && len(e.discoveryTtl) > 0 { <-e.discoveryTtl servers, ttl, err := discoverDNS(e.DiscoveryZone) if err != nil { From 66e146c78fdfb87d3cc692b5f9efebd2c93813da Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Wed, 18 Jun 2014 10:09:29 -0500 Subject: [PATCH 6/8] compose serviceurls after discovering boxes via DNS --- connection.go | 3 ++- dns_discover.go | 9 ++++++--- dns_discover_test.go | 4 ++-- struct.go | 1 + 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/connection.go b/connection.go index 495016c..aaad881 100644 --- a/connection.go +++ b/connection.go @@ -17,7 +17,7 @@ func init() { func (e *EurekaConnection) SelectServiceURL() string { if e.DNSDiscovery && len(e.discoveryTtl) > 0 { <-e.discoveryTtl - servers, ttl, err := discoverDNS(e.DiscoveryZone) + servers, ttl, err := discoverDNS(e.DiscoveryZone, e.ServicePort) if err != nil { return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] } @@ -42,6 +42,7 @@ func NewConnFromConfigFile(location string) (c EurekaConnection, err error) { // those options func NewConnFromConfig(conf Config) (c EurekaConnection) { c.ServiceUrls = conf.Eureka.ServiceUrls + c.ServicePort = conf.Eureka.ServerPort if len(c.ServiceUrls) == 0 && len(conf.Eureka.ServerDNSName) > 0 { c.ServiceUrls = []string{conf.Eureka.ServerDNSName} } diff --git a/dns_discover.go b/dns_discover.go index 4e9c9f7..47fb298 100644 --- a/dns_discover.go +++ b/dns_discover.go @@ -13,7 +13,7 @@ const azURL = "http://169.254.169.254/latest/meta-data/placement/availability-zo var ErrNotInAWS = fmt.Errorf("Not in AWS") -func discoverDNS(domain string) (servers []string, ttl time.Duration, err error) { +func discoverDNS(domain string, port int) (servers []string, ttl time.Duration, err error) { r, _ := region() // all DNS queries must use the FQDN @@ -28,11 +28,14 @@ func discoverDNS(domain string) (servers []string, ttl time.Duration, err error) } for _, az := range regionRecords { - s, _, er := findTXT("txt." + dns.Fqdn(az)) + instances, _, er := findTXT("txt." + dns.Fqdn(az)) if er != nil { continue } - servers = append(servers, s...) + for _, instance := range instances { + // format the service URL + servers = append(servers, fmt.Sprintf("http://%s:%s/eureka/v2/", instance, port)) + } } return } diff --git a/dns_discover_test.go b/dns_discover_test.go index f1b6414..bdff59d 100644 --- a/dns_discover_test.go +++ b/dns_discover_test.go @@ -46,13 +46,13 @@ func TestGetNetflixTestDomain(t *testing.T) { }) }) Convey("Autodiscover discoverytest.netflix.net.", t, func() { - servers, ttl, err := discoverDNS("discoverytest.netflix.net") + servers, ttl, err := discoverDNS("discoverytest.netflix.net", 7001) So(ttl, ShouldEqual, 60*time.Second) So(err, ShouldBeNil) So(len(servers), ShouldEqual, 4) Convey("Servers discovered should all be EC2 DNS names", func() { for _, s := range servers { - So(s[0:4], ShouldEqual, "ec2-") + So(s[0:11], ShouldEqual, "http://ec2-") } }) }) diff --git a/struct.go b/struct.go index 0e8b511..968acbe 100644 --- a/struct.go +++ b/struct.go @@ -15,6 +15,7 @@ var EurekaURLSlugs = map[string]string{ // EurekaConnection is the settings required to make eureka requests type EurekaConnection struct { ServiceUrls []string + ServicePort int Timeout time.Duration PollInterval time.Duration PreferSameZone bool From f8de9676fd223ba69e086b8b4120fd3922d8f06f Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Thu, 19 Jun 2014 12:40:00 -0500 Subject: [PATCH 7/8] change the way that DNS TTLs are handled --- connection.go | 13 ++++++++++--- dns_discover.go | 2 +- struct.go | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/connection.go b/connection.go index aaad881..38998d5 100644 --- a/connection.go +++ b/connection.go @@ -15,13 +15,20 @@ func init() { // balancing scheme. // TODO: Make this not just pick a random one. func (e *EurekaConnection) SelectServiceURL() string { - if e.DNSDiscovery && len(e.discoveryTtl) > 0 { - <-e.discoveryTtl + if e.discoveryTtl == nil { + e.discoveryTtl = make(chan struct{}, 1) + } + if e.DNSDiscovery && len(e.discoveryTtl) == 0 { servers, ttl, err := discoverDNS(e.DiscoveryZone, e.ServicePort) if err != nil { return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] } - e.discoveryTtl = time.After(ttl) + e.discoveryTtl <- struct{}{} + time.AfterFunc(ttl, func() { + // At the end of the timeout, empty the channel so that the next + // SelectServiceURL call will refresh the DNS info + <-e.discoveryTtl + }) e.ServiceUrls = servers } return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] diff --git a/dns_discover.go b/dns_discover.go index 47fb298..67885bc 100644 --- a/dns_discover.go +++ b/dns_discover.go @@ -34,7 +34,7 @@ func discoverDNS(domain string, port int) (servers []string, ttl time.Duration, } for _, instance := range instances { // format the service URL - servers = append(servers, fmt.Sprintf("http://%s:%s/eureka/v2/", instance, port)) + servers = append(servers, fmt.Sprintf("http://%s:%d/eureka/v2", instance, port)) } } return diff --git a/struct.go b/struct.go index 968acbe..1de496e 100644 --- a/struct.go +++ b/struct.go @@ -22,7 +22,7 @@ type EurekaConnection struct { Retries int DNSDiscovery bool DiscoveryZone string - discoveryTtl <-chan time.Time + discoveryTtl chan struct{} } // GetAppsResponse lets us deserialize the eureka/v2/apps response XML From f67c16713165564901d562c19063314dcfbac534 Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Thu, 19 Jun 2014 17:35:27 -0500 Subject: [PATCH 8/8] pull out repeated selection logic --- connection.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/connection.go b/connection.go index 04d395c..3531f38 100644 --- a/connection.go +++ b/connection.go @@ -21,7 +21,7 @@ func (e *EurekaConnection) SelectServiceURL() string { if e.DNSDiscovery && len(e.discoveryTtl) == 0 { servers, ttl, err := discoverDNS(e.DiscoveryZone, e.ServicePort) if err != nil { - return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] + return choice(e.ServiceUrls) } e.discoveryTtl <- struct{}{} time.AfterFunc(ttl, func() { @@ -31,7 +31,11 @@ func (e *EurekaConnection) SelectServiceURL() string { }) e.ServiceUrls = servers } - return e.ServiceUrls[rand.Int()%len(e.ServiceUrls)] + return choice(e.ServiceUrls) +} + +func choice(options []string) string { + return options[rand.Int()%len(options)] } // NewConnFromConfigFile sets up a connection object based on a config in