diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..11461a1d85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +aws-k8s-agent +aws-cni +verify-aws +verify-network +Gopkg.lock diff --git a/Gopkg.lock b/Gopkg.lock index 63c30739a3..46b8ae134a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -13,6 +13,12 @@ packages = ["."] revision = "de5bf2ad457846296e2031421a34e2568e304e35" +[[projects]] + name = "github.com/aws/amazon-ecs-agent" + packages = ["agent/utils/ttime","agent/utils/ttime/mocks"] + revision = "ea58fddf8462034ff2034aa9788b02341d7702e3" + version = "v1.15.2" + [[projects]] branch = "master" name = "github.com/aws/amazon-ecs-cni-plugins" @@ -237,6 +243,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1003f2048943d3eee6747174022a8e41f8e6b93408bff2f636ea53ea62e0e696" + inputs-digest = "b6c35664386e0bd1130101921cf7b02047e942892491d64f3bac3507f9146f02" solver-name = "gps-cdcl" solver-version = 1 diff --git a/ipamd/datastore/data_store.go b/ipamd/datastore/data_store.go index 908500e0ef..4ffd0b7fd2 100644 --- a/ipamd/datastore/data_store.go +++ b/ipamd/datastore/data_store.go @@ -26,47 +26,52 @@ import ( const ( minLifeTime = 1 * time.Minute addressCoolingPeriod = 1 * time.Minute - // DuplicatedENIError is an error when caller tries to add an duplicate eni to data store - DuplicatedENIError = "data store: duplicate eni" + // DuplicatedENIError is an error when caller tries to add an duplicate ENI to data store + DuplicatedENIError = "data store: duplicate ENI" - // DuplicateIPError is an error when caller tries to add an duplicate ip address to data store + // DuplicateIPError is an error when caller tries to add an duplicate IP address to data store DuplicateIPError = "datastore: duplicated IP" ) // ErrUnknownPod is an error when there is no pod in data store matching pod name, namespace, container id var ErrUnknownPod = errors.New("datastore: unknown pod") -// ErrUnknownPodIP is an error where pod's ip address is not found in data store -var ErrUnknownPodIP = errors.New("datastore: pod using unknown ip address") +// ErrUnknownPodIP is an error where pod's IP address is not found in data store +var ErrUnknownPodIP = errors.New("datastore: pod using unknown IP address") -// ENIIPPool contains ENI/IP Pool information +// ENIIPPool contains ENI/IP Pool information. Exported fields will be Marshaled for introspection. type ENIIPPool struct { - createTime time.Time - lastUnassignedTime time.Time - isPrimary bool - id string - deviceNumber int - assignedIPv4Addresses int - ipv4Addresses map[string]*AddressInfo + createTime time.Time + lastUnAssignedTime time.Time + // IsPrimary indicates whether ENI is a primary ENI + IsPrimary bool + id string + // DeviceNumber is the device number of ENI + DeviceNumber int + // AssignedIPv4Addresses is the number of IP addesses already been assigned + AssignedIPv4Addresses int + // IPv4Addresses shows whether each address is assigned, the key is IP address, which must + // be in dot-decimal notation with no leading zeros and no whitespace(eg: "10.1.0.253") + IPv4Addresses map[string]*AddressInfo } -// AddressInfo contains inforation about an IP +// AddressInfo contains inforation about an IP, Exported fields will be Marshaled for introspection. type AddressInfo struct { address string - assigned bool // true if it is assigned to a Pod + Assigned bool // true if it is assigned to a pod unAssignedTime time.Time } -// PodKey is used to locate Pod IP +// PodKey is used to locate pod IP type PodKey struct { name string namespace string container string } -// PodIPInfo contains Pod's IP and the device number of the ENI +// PodIPInfo contains pod's IP and the device number of the ENI type PodIPInfo struct { - // IP is the ip address of pod + // IP is the IP address of pod IP string // DeviceNumber is the device number of pod DeviceNumber int @@ -81,6 +86,19 @@ type DataStore struct { lock sync.RWMutex } +// PodInfos contains pods IP information which uses key name_namespace_container +type PodInfos map[string]PodIPInfo + +// ENIInfos contains ENI IP information +type ENIInfos struct { + // TotalIPs is the total number of IP addresses + TotalIPs int + // assigned is the number of IP addresses that has been assigned + AssignedIPs int + // ENIIPPools contains ENI IP pool information + ENIIPPools map[string]ENIIPPool +} + // NewDataStore returns DataStore structure func NewDataStore() *DataStore { return &DataStore{ @@ -89,7 +107,7 @@ func NewDataStore() *DataStore { } } -// AddENI add eni to data store +// AddENI add ENI to data store func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) error { ds.lock.Lock() defer ds.lock.Unlock() @@ -102,10 +120,10 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro } ds.eniIPPools[eniID] = &ENIIPPool{ createTime: time.Now(), - isPrimary: isPrimary, + IsPrimary: isPrimary, id: eniID, - deviceNumber: deviceNumber, - ipv4Addresses: make(map[string]*AddressInfo)} + DeviceNumber: deviceNumber, + IPv4Addresses: make(map[string]*AddressInfo)} return nil } @@ -120,24 +138,24 @@ func (ds *DataStore) AddENIIPv4Address(eniID string, ipv4 string) error { curENI, ok := ds.eniIPPools[eniID] if !ok { - return errors.New("add eni's ip to datastore: unknown eni") + return errors.New("add ENI's IP to datastore: unknown ENI") } - _, ok = curENI.ipv4Addresses[ipv4] + _, ok = curENI.IPv4Addresses[ipv4] if ok { return errors.New(DuplicateIPError) } ds.total++ - curENI.ipv4Addresses[ipv4] = &AddressInfo{address: ipv4, assigned: false} + curENI.IPv4Addresses[ipv4] = &AddressInfo{address: ipv4, Assigned: false} - log.Infof("Added eni(%s)'s ip %s to datastore", eniID, ipv4) + log.Infof("Added ENI(%s)'s IP %s to datastore", eniID, ipv4) return nil } -// AssignPodIPv4Address assigns an IPv4 address to Pod +// AssignPodIPv4Address assigns an IPv4 address to pod // It returns the assigned IPv4 address, device number, error func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, int, error) { ds.lock.Lock() @@ -154,13 +172,13 @@ func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, in if ok { if ipAddr.IP == k8sPod.IP && k8sPod.IP != "" { // The caller invoke multiple times to assign(PodName/NameSpace --> same IPAddress). It is not a error, but not very efficient. - log.Infof("AssignPodIPv4Address: duplicate pod assign for ip %s, name %s, namespace %s, container %s", + log.Infof("AssignPodIPv4Address: duplicate pod assign for IP %s, name %s, namespace %s, container %s", k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) return ipAddr.IP, ipAddr.DeviceNumber, nil } //TODO handle this bug assert?, may need to add a counter here, if counter is too high, need to mark node as unhealthy... // this is a bug that the caller invoke multiple times to assign(PodName/NameSpace -> a different IPaddress). - log.Errorf("AssignPodIPv4Address: current ip %s is changed to ip %s for POD(name %s, namespace %s, container %s)", + log.Errorf("AssignPodIPv4Address: current IP %s is changed to IP %s for pod(name %s, namespace %s, container %s)", ipAddr, k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) return "", 0, errors.New("datastore; invalid pod with multiple IP addresses") @@ -177,33 +195,33 @@ func (ds *DataStore) assignPodIPv4AddressUnsafe(k8sPod *k8sapi.K8SPodInfo) (stri container: k8sPod.Container, } for _, eni := range ds.eniIPPools { - if (k8sPod.IP == "") && (len(eni.ipv4Addresses) == eni.assignedIPv4Addresses) { - // skip this eni, since it has no available ip address + if (k8sPod.IP == "") && (len(eni.IPv4Addresses) == eni.AssignedIPv4Addresses) { + // skip this ENI, since it has no available IP address log.Debugf("AssignPodIPv4Address, skip ENI %s that do not have available addresses", eni.id) continue } - for _, addr := range eni.ipv4Addresses { + for _, addr := range eni.IPv4Addresses { if k8sPod.IP == addr.address { - // After L-IPAM restart and built IP warm-pool, it needs to take the existing running POD IP out of the pool. - if !addr.assigned { + // After L-IPAM restart and built IP warm-pool, it needs to take the existing running pod IP out of the pool. + if !addr.Assigned { ds.assigned++ - eni.assignedIPv4Addresses++ - addr.assigned = true + eni.AssignedIPv4Addresses++ + addr.Assigned = true } - ds.podsIP[podKey] = PodIPInfo{IP: addr.address, DeviceNumber: eni.deviceNumber} - log.Infof("AssignPodIPv4Address Reassign IP %v to Pod (name %s, namespace %s)", + ds.podsIP[podKey] = PodIPInfo{IP: addr.address, DeviceNumber: eni.DeviceNumber} + log.Infof("AssignPodIPv4Address Reassign IP %v to pod (name %s, namespace %s)", addr.address, k8sPod.Name, k8sPod.Namespace) - return addr.address, eni.deviceNumber, nil + return addr.address, eni.DeviceNumber, nil } - if !addr.assigned && k8sPod.IP == "" { - // This is triggered by a POD's Add Network command from CNI plugin + if !addr.Assigned && k8sPod.IP == "" { + // This is triggered by a pod's Add Network command from CNI plugin ds.assigned++ - eni.assignedIPv4Addresses++ - addr.assigned = true - ds.podsIP[podKey] = PodIPInfo{IP: addr.address, DeviceNumber: eni.deviceNumber} - log.Infof("AssignPodIPv4Address Assign IP %v to Pod (name %s, namespace %s container %s)", + eni.AssignedIPv4Addresses++ + addr.Assigned = true + ds.podsIP[podKey] = PodIPInfo{IP: addr.address, DeviceNumber: eni.DeviceNumber} + log.Infof("AssignPodIPv4Address Assign IP %v to pod (name %s, namespace %s container %s)", addr.address, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) - return addr.address, eni.deviceNumber, nil + return addr.address, eni.DeviceNumber, nil } } @@ -222,7 +240,7 @@ func (ds *DataStore) GetStats() (int, int) { func (ds *DataStore) getDeletableENI() *ENIIPPool { for _, eni := range ds.eniIPPools { - if eni.isPrimary { + if eni.IsPrimary { continue } @@ -230,11 +248,11 @@ func (ds *DataStore) getDeletableENI() *ENIIPPool { continue } - if time.Now().Sub(eni.lastUnassignedTime) < addressCoolingPeriod { + if time.Now().Sub(eni.lastUnAssignedTime) < addressCoolingPeriod { continue } - if eni.assignedIPv4Addresses != 0 { + if eni.AssignedIPv4Addresses != 0 { continue } @@ -252,14 +270,14 @@ func (ds *DataStore) FreeENI() (string, error) { deletableENI := ds.getDeletableENI() if deletableENI == nil { - log.Debugf("FreeENI: no deletable ENI") - return "", errors.New("free eni: none of enis can be deleted at this time") + log.Debugf("No ENI can be deleted at this time") + return "", errors.New("free ENI: no ENI can be deleted at this time") } - ds.total -= len(ds.eniIPPools[deletableENI.id].ipv4Addresses) - ds.assigned -= deletableENI.assignedIPv4Addresses + ds.total -= len(ds.eniIPPools[deletableENI.id].IPv4Addresses) + ds.assigned -= deletableENI.AssignedIPv4Addresses log.Infof("FreeENI %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", - deletableENI.id, len(ds.eniIPPools[deletableENI.id].ipv4Addresses), ds.total, ds.assigned) + deletableENI.id, len(ds.eniIPPools[deletableENI.id].IPv4Addresses), ds.total, ds.assigned) deletedENI := deletableENI.id delete(ds.eniIPPools, deletableENI.id) @@ -267,11 +285,11 @@ func (ds *DataStore) FreeENI() (string, error) { } // UnAssignPodIPv4Address a) find out the IP address based on PodName and PodNameSpace -// b) mark IP address as unassigned c) returns IP address, eni's device number, error +// b) mark IP address as unassigned c) returns IP address, ENI's device number, error func (ds *DataStore) UnAssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, int, error) { ds.lock.Lock() defer ds.lock.Unlock() - log.Debugf("UnAssignIPv4Address: IP address pool stats: total:%d, assigned %d, Pod(Name: %s, Namespace: %s, Container %s)", + log.Debugf("UnAssignIPv4Address: IP address pool stats: total:%d, assigned %d, pod(Name: %s, Namespace: %s, Container %s)", ds.total, ds.assigned, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) podKey := PodKey{ @@ -287,22 +305,57 @@ func (ds *DataStore) UnAssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, } for _, eni := range ds.eniIPPools { - ip, ok := eni.ipv4Addresses[ipAddr.IP] - if ok && ip.assigned { - ip.assigned = false + ip, ok := eni.IPv4Addresses[ipAddr.IP] + if ok && ip.Assigned { + ip.Assigned = false ds.assigned-- - eni.assignedIPv4Addresses-- + eni.AssignedIPv4Addresses-- curTime := time.Now() ip.unAssignedTime = curTime - eni.lastUnassignedTime = curTime - log.Infof("UnAssignIPv4Address: Pod (Name: %s, NameSpace %s Container %s)'s ipAddr %s, DeviceNumber%d", - k8sPod.Name, k8sPod.Namespace, k8sPod.Container, ip.address, eni.deviceNumber) + eni.lastUnAssignedTime = curTime + log.Infof("UnAssignIPv4Address: pod (Name: %s, NameSpace %s Container %s)'s ipAddr %s, DeviceNumber%d", + k8sPod.Name, k8sPod.Namespace, k8sPod.Container, ip.address, eni.DeviceNumber) delete(ds.podsIP, podKey) - return ip.address, eni.deviceNumber, nil + return ip.address, eni.DeviceNumber, nil } } - log.Warnf("UnassignIPv4Address: Failed to find pod %s namespace %s Container %s using ip %s", + log.Warnf("UnassignIPv4Address: Failed to find pod %s namespace %s container %s using IP %s", k8sPod.Name, k8sPod.Namespace, k8sPod.Container, ipAddr.IP) return "", 0, ErrUnknownPodIP } + +// GetPodInfos provides pod IP information to introspection endpoint +func (ds *DataStore) GetPodInfos() *map[string]PodIPInfo { + ds.lock.Lock() + defer ds.lock.Unlock() + + var podInfos = make(map[string]PodIPInfo, len(ds.podsIP)) + + for podKey, podInfo := range ds.podsIP { + key := podKey.name + "_" + podKey.namespace + "_" + podKey.container + podInfos[key] = podInfo + log.Debugf("introspect: key %s", key) + } + + log.Debugf("introspect: len %d", len(ds.podsIP)) + + return &podInfos +} + +// GetENIInfos provides ENI IP information to introspection endpoint +func (ds *DataStore) GetENIInfos() *ENIInfos { + ds.lock.Lock() + defer ds.lock.Unlock() + + var eniInfos = ENIInfos{ + TotalIPs: ds.total, + AssignedIPs: ds.assigned, + ENIIPPools: make(map[string]ENIIPPool, len(ds.eniIPPools)), + } + + for eni, eniInfo := range ds.eniIPPools { + eniInfos.ENIIPPools[eni] = *eniInfo + } + return &eniInfos +} diff --git a/ipamd/datastore/data_store_test.go b/ipamd/datastore/data_store_test.go index 3e736d853a..031a7099c7 100644 --- a/ipamd/datastore/data_store_test.go +++ b/ipamd/datastore/data_store_test.go @@ -34,6 +34,9 @@ func TestAddENI(t *testing.T) { assert.NoError(t, err) assert.Equal(t, len(ds.eniIPPools), 2) + + eniInfos := ds.GetENIInfos() + assert.Equal(t, len(eniInfos.ENIIPPools), 2) } func TestAddENIIPv4Address(t *testing.T) { @@ -48,29 +51,29 @@ func TestAddENIIPv4Address(t *testing.T) { err = ds.AddENIIPv4Address("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 1) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) err = ds.AddENIIPv4Address("eni-1", "1.1.1.1") assert.Error(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 1) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) err = ds.AddENIIPv4Address("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 2) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) err = ds.AddENIIPv4Address("eni-2", "1.1.2.2") assert.NoError(t, err) assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].ipv4Addresses), 1) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) err = ds.AddENIIPv4Address("dummy-eni", "1.1.2.2") assert.Error(t, err) assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].ipv4Addresses), 1) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) } @@ -98,17 +101,20 @@ func TestPodIPv4Address(t *testing.T) { assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.1") assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 2) - assert.Equal(t, ds.eniIPPools["eni-1"].assignedIPv4Addresses, 1) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, ds.eniIPPools["eni-1"].AssignedIPv4Addresses, 1) ip, _, err = ds.AssignPodIPv4Address(&podInfo) + podsInfos := ds.GetPodInfos() + assert.Equal(t, len(*podsInfos), 1) + // duplicate add ip, _, err = ds.AssignPodIPv4Address(&podInfo) assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.1") assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 2) - assert.Equal(t, ds.eniIPPools["eni-1"].assignedIPv4Addresses, 1) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, ds.eniIPPools["eni-1"].AssignedIPv4Addresses, 1) // wrong ip address podInfo = k8sapi.K8SPodInfo{ @@ -131,8 +137,11 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, ip, "1.1.2.2") assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].ipv4Addresses), 1) - assert.Equal(t, ds.eniIPPools["eni-2"].assignedIPv4Addresses, 1) + assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) + assert.Equal(t, ds.eniIPPools["eni-2"].AssignedIPv4Addresses, 1) + + podsInfos = ds.GetPodInfos() + assert.Equal(t, len(*podsInfos), 2) podInfo = k8sapi.K8SPodInfo{ Name: "pod-1", @@ -145,8 +154,8 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, ip, "1.1.1.2") assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].ipv4Addresses), 2) - assert.Equal(t, ds.eniIPPools["eni-1"].assignedIPv4Addresses, 2) + assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, ds.eniIPPools["eni-1"].AssignedIPv4Addresses, 2) // no more IP addresses podInfo = k8sapi.K8SPodInfo{ @@ -179,15 +188,15 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 2) assert.Equal(t, deviceNum, pod1Ns2Device) - assert.Equal(t, len(ds.eniIPPools["eni-2"].ipv4Addresses), 1) - assert.Equal(t, ds.eniIPPools["eni-2"].assignedIPv4Addresses, 0) + assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) + assert.Equal(t, ds.eniIPPools["eni-2"].AssignedIPv4Addresses, 0) // should not able to free this eni eni, _ := ds.FreeENI() assert.True(t, (eni == "")) ds.eniIPPools["eni-2"].createTime = time.Time{} - ds.eniIPPools["eni-2"].lastUnassignedTime = time.Time{} + ds.eniIPPools["eni-2"].lastUnAssignedTime = time.Time{} eni, _ = ds.FreeENI() assert.Equal(t, eni, "eni-2") diff --git a/ipamd/introspect.go b/ipamd/introspect.go new file mode 100644 index 0000000000..59490bbd60 --- /dev/null +++ b/ipamd/introspect.go @@ -0,0 +1,131 @@ +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ipamd + +import ( + "encoding/json" + "net/http" + "strconv" + "sync" + "time" + + log "github.com/cihub/seelog" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils" +) + +const ( + // IntrospectionPort is the port for ipamd introspection + IntrospectionPort = 51678 +) + +type rootResponse struct { + AvailableCommands []string +} + +// LoggingHandler is a object for handling http request +type LoggingHandler struct{ h http.Handler } + +// NewLoggingHandler creates a new LoggingHandler object. +func NewLoggingHandler(handler http.Handler) LoggingHandler { + return LoggingHandler{h: handler} +} + +func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log.Info("Handling http request", "method", r.Method, "from", r.RemoteAddr, "uri", r.RequestURI) + lh.h.ServeHTTP(w, r) +} + +// SetupHTTP sets up ipamd introspection service endpoint +func (c *IPAMContext) SetupHTTP() { + server := c.setupServer() + + for { + once := sync.Once{} + utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error { + // TODO, make this cancellable and use the passed in context; for + // now, not critical if this gets interrupted + err := server.ListenAndServe() + once.Do(func() { + log.Error("Error running http api", "err", err) + }) + return err + }) + } +} + +func (c *IPAMContext) setupServer() *http.Server { + serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){ + "/v1/enis": eniV1RequestHandler(c), + "/v1/pods": podV1RequestHandler(c), + } + paths := make([]string, 0, len(serverFunctions)) + for path := range serverFunctions { + paths = append(paths, path) + } + availableCommands := &rootResponse{paths} + // Autogenerated list of the above serverFunctions paths + availableCommandResponse, err := json.Marshal(&availableCommands) + + if err != nil { + log.Error("Failed to Marshal: %v", err) + } + + defaultHandler := func(w http.ResponseWriter, r *http.Request) { + w.Write(availableCommandResponse) + } + + serveMux := http.NewServeMux() + serveMux.HandleFunc("/", defaultHandler) + for key, fn := range serverFunctions { + serveMux.HandleFunc(key, fn) + } + + // Log all requests and then pass through to serveMux + loggingServeMux := http.NewServeMux() + loggingServeMux.Handle("/", LoggingHandler{serveMux}) + + server := &http.Server{ + Addr: ":" + strconv.Itoa(IntrospectionPort), + Handler: loggingServeMux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + } + + return server +} + +func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + responseJSON, err := json.Marshal(ipam.dataStore.GetENIInfos()) + if err != nil { + log.Error("Failed to marshal ENI data: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + w.Write(responseJSON) + } +} + +func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + responseJSON, err := json.Marshal(ipam.dataStore.GetPodInfos()) + if err != nil { + log.Error("Failed to marshal pod data: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + w.Write(responseJSON) + } +} diff --git a/main.go b/main.go index 109ba88f0c..9144403537 100644 --- a/main.go +++ b/main.go @@ -40,5 +40,6 @@ func main() { } go aws_k8s_agent.StartNodeIPPoolManager() + go aws_k8s_agent.SetupHTTP() aws_k8s_agent.RunRPCHandler() } diff --git a/pkg/utils/backoff.go b/pkg/utils/backoff.go new file mode 100644 index 0000000000..1e65fe5f43 --- /dev/null +++ b/pkg/utils/backoff.go @@ -0,0 +1,80 @@ +// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package utils + +//TODO needs to extract this to library (it's copied from ecs agent) +import ( + "math" + "math/rand" + "sync" + "time" +) + +type Backoff interface { + Reset() + Duration() time.Duration +} + +type SimpleBackoff struct { + current time.Duration + start time.Duration + max time.Duration + jitterMultiple float64 + multiple float64 + mu sync.Mutex +} + +// NewSimpleBackoff creates a Backoff which ranges from min to max increasing by +// multiple each time. +// It also adds (and yes, the jitter is always added, never +// subtracted) a random amount of jitter up to jitterMultiple percent (that is, +// jitterMultiple = 0.0 is no jitter, 0.15 is 15% added jitter). The total time +// may exceed "max" when accounting for jitter, such that the absolute max is +// max + max * jiterMultiple +func NewSimpleBackoff(min, max time.Duration, jitterMultiple, multiple float64) *SimpleBackoff { + return &SimpleBackoff{ + start: min, + current: min, + max: max, + jitterMultiple: jitterMultiple, + multiple: multiple, + } +} + +func (sb *SimpleBackoff) Duration() time.Duration { + sb.mu.Lock() + defer sb.mu.Unlock() + ret := sb.current + sb.current = time.Duration(math.Min(float64(sb.max.Nanoseconds()), float64(float64(sb.current.Nanoseconds())*sb.multiple))) + + return AddJitter(ret, time.Duration(int64(float64(ret)*sb.jitterMultiple))) +} + +func (sb *SimpleBackoff) Reset() { + sb.mu.Lock() + defer sb.mu.Unlock() + sb.current = sb.start +} + +// AddJitter adds an amount of jitter between 0 and the given jitter to the +// given duration +func AddJitter(duration time.Duration, jitter time.Duration) time.Duration { + var randJitter int64 + if jitter.Nanoseconds() == 0 { + randJitter = 0 + } else { + randJitter = rand.Int63n(jitter.Nanoseconds()) + } + return time.Duration(duration.Nanoseconds() + randJitter) +} diff --git a/pkg/utils/backoff_test.go b/pkg/utils/backoff_test.go new file mode 100644 index 0000000000..1c356bbc0a --- /dev/null +++ b/pkg/utils/backoff_test.go @@ -0,0 +1,51 @@ +// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package utils + +import ( + "testing" + "time" +) + +func TestSimpleBackoff(t *testing.T) { + sb := NewSimpleBackoff(10*time.Second, time.Minute, 0, 2) + + for i := 0; i < 2; i++ { + duration := sb.Duration() + if duration.Nanoseconds() != 10*time.Second.Nanoseconds() { + t.Error("Initial duration incorrect. Got ", duration.Nanoseconds()) + } + + duration = sb.Duration() + if duration.Nanoseconds() != 20*time.Second.Nanoseconds() { + t.Error("Increase incorrect") + } + _ = sb.Duration() // 40s + duration = sb.Duration() + if duration.Nanoseconds() != 60*time.Second.Nanoseconds() { + t.Error("Didn't stop at maximum") + } + sb.Reset() + // loop to redo the above tests after resetting, they should be the same + } +} + +func TestJitter(t *testing.T) { + for i := 0; i < 10; i++ { + duration := AddJitter(10*time.Second, 3*time.Second) + if duration < 10*time.Second || duration > 13*time.Second { + t.Error("Excessive amount of jitter", duration) + } + } +} diff --git a/pkg/utils/errors.go b/pkg/utils/errors.go new file mode 100644 index 0000000000..98005ccb97 --- /dev/null +++ b/pkg/utils/errors.go @@ -0,0 +1,90 @@ +// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package utils + +import ( + "fmt" + "strings" +) + +type Retriable interface { + Retry() bool +} + +type DefaultRetriable struct { + retry bool +} + +func (dr DefaultRetriable) Retry() bool { + return dr.retry +} + +func NewRetriable(retry bool) Retriable { + return DefaultRetriable{ + retry: retry, + } +} + +type RetriableError interface { + Retriable + error +} + +type DefaultRetriableError struct { + Retriable + error +} + +func NewRetriableError(retriable Retriable, err error) RetriableError { + return &DefaultRetriableError{ + retriable, + err, + } +} + +type AttributeError struct { + err string +} + +func (e AttributeError) Error() string { + return e.err +} + +func NewAttributeError(err string) AttributeError { + return AttributeError{err} +} + +// Implements error +type MultiErr struct { + errors []error +} + +func (me MultiErr) Error() string { + ret := make([]string, len(me.errors)+1) + ret[0] = "Multiple error:" + for ndx, err := range me.errors { + ret[ndx+1] = fmt.Sprintf("\t%d: %s", ndx, err.Error()) + } + return strings.Join(ret, "\n") +} + +func NewMultiError(errs ...error) error { + errors := make([]error, 0, len(errs)) + for _, err := range errs { + if err != nil { + errors = append(errors, err) + } + } + return MultiErr{errors} +} diff --git a/pkg/utils/ttime/generate_mocks.go b/pkg/utils/ttime/generate_mocks.go new file mode 100644 index 0000000000..39c90af158 --- /dev/null +++ b/pkg/utils/ttime/generate_mocks.go @@ -0,0 +1,16 @@ +// Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ttime + +//go:generate go run ../../../scripts/mockgen.go github.com/aws/amazon-vpc-cni-k8s/pkg/utils/ttime Time,Timer mocks/time_mocks.go diff --git a/pkg/utils/ttime/mocks/time_mocks.go b/pkg/utils/ttime/mocks/time_mocks.go new file mode 100644 index 0000000000..d8d19f11bb --- /dev/null +++ b/pkg/utils/ttime/mocks/time_mocks.go @@ -0,0 +1,142 @@ +// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/utils/ttime (interfaces: Time,Timer) + +// Package mock_ttime is a generated GoMock package. +package mock_ttime + +import ( + reflect "reflect" + time "time" + + ttime "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/ttime" + gomock "github.com/golang/mock/gomock" +) + +// MockTime is a mock of Time interface +type MockTime struct { + ctrl *gomock.Controller + recorder *MockTimeMockRecorder +} + +// MockTimeMockRecorder is the mock recorder for MockTime +type MockTimeMockRecorder struct { + mock *MockTime +} + +// NewMockTime creates a new mock instance +func NewMockTime(ctrl *gomock.Controller) *MockTime { + mock := &MockTime{ctrl: ctrl} + mock.recorder = &MockTimeMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockTime) EXPECT() *MockTimeMockRecorder { + return m.recorder +} + +// After mocks base method +func (m *MockTime) After(arg0 time.Duration) <-chan time.Time { + ret := m.ctrl.Call(m, "After", arg0) + ret0, _ := ret[0].(<-chan time.Time) + return ret0 +} + +// After indicates an expected call of After +func (mr *MockTimeMockRecorder) After(arg0 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "After", reflect.TypeOf((*MockTime)(nil).After), arg0) +} + +// AfterFunc mocks base method +func (m *MockTime) AfterFunc(arg0 time.Duration, arg1 func()) ttime.Timer { + ret := m.ctrl.Call(m, "AfterFunc", arg0, arg1) + ret0, _ := ret[0].(ttime.Timer) + return ret0 +} + +// AfterFunc indicates an expected call of AfterFunc +func (mr *MockTimeMockRecorder) AfterFunc(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AfterFunc", reflect.TypeOf((*MockTime)(nil).AfterFunc), arg0, arg1) +} + +// Now mocks base method +func (m *MockTime) Now() time.Time { + ret := m.ctrl.Call(m, "Now") + ret0, _ := ret[0].(time.Time) + return ret0 +} + +// Now indicates an expected call of Now +func (mr *MockTimeMockRecorder) Now() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Now", reflect.TypeOf((*MockTime)(nil).Now)) +} + +// Sleep mocks base method +func (m *MockTime) Sleep(arg0 time.Duration) { + m.ctrl.Call(m, "Sleep", arg0) +} + +// Sleep indicates an expected call of Sleep +func (mr *MockTimeMockRecorder) Sleep(arg0 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sleep", reflect.TypeOf((*MockTime)(nil).Sleep), arg0) +} + +// MockTimer is a mock of Timer interface +type MockTimer struct { + ctrl *gomock.Controller + recorder *MockTimerMockRecorder +} + +// MockTimerMockRecorder is the mock recorder for MockTimer +type MockTimerMockRecorder struct { + mock *MockTimer +} + +// NewMockTimer creates a new mock instance +func NewMockTimer(ctrl *gomock.Controller) *MockTimer { + mock := &MockTimer{ctrl: ctrl} + mock.recorder = &MockTimerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockTimer) EXPECT() *MockTimerMockRecorder { + return m.recorder +} + +// Reset mocks base method +func (m *MockTimer) Reset(arg0 time.Duration) bool { + ret := m.ctrl.Call(m, "Reset", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// Reset indicates an expected call of Reset +func (mr *MockTimerMockRecorder) Reset(arg0 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockTimer)(nil).Reset), arg0) +} + +// Stop mocks base method +func (m *MockTimer) Stop() bool { + ret := m.ctrl.Call(m, "Stop") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Stop indicates an expected call of Stop +func (mr *MockTimerMockRecorder) Stop() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockTimer)(nil).Stop)) +} diff --git a/pkg/utils/ttime/ttime.go b/pkg/utils/ttime/ttime.go new file mode 100644 index 0000000000..485aed7134 --- /dev/null +++ b/pkg/utils/ttime/ttime.go @@ -0,0 +1,60 @@ +// Package ttime implements a testable alternative to the Go "time" package. +package ttime + +import "time" + +// Time represents an implementation for this package's methods +type Time interface { + Now() time.Time + Sleep(d time.Duration) + After(d time.Duration) <-chan time.Time + AfterFunc(d time.Duration, f func()) Timer +} + +type Timer interface { + Reset(d time.Duration) bool + Stop() bool +} + +// DefaultTime is a Time that behaves normally +type DefaultTime struct{} + +var _time Time = &DefaultTime{} + +// Now returns the current time +func (*DefaultTime) Now() time.Time { + return time.Now() +} + +// Sleep sleeps for the given duration +func (*DefaultTime) Sleep(d time.Duration) { + time.Sleep(d) +} + +// After sleeps for the given duration and then writes to to the returned channel +func (*DefaultTime) After(d time.Duration) <-chan time.Time { + return time.After(d) +} + +// AfterFunc waits for the duration to elapse and then calls f in its own +// goroutine. It returns a Timer that can be used to cancel the call using its +// Stop method. +func (*DefaultTime) AfterFunc(d time.Duration, f func()) Timer { + return time.AfterFunc(d, f) +} + +// SetTime configures what 'Time' implementation to use for each of the +// package-level methods. +func SetTime(t Time) { + _time = t +} + +// Now returns the implementation's current time +func Now() time.Time { + return _time.Now() +} + +// Since returns the time different from Now and the given time t +func Since(t time.Time) time.Duration { + return _time.Now().Sub(t) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000000..33f021f38b --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,198 @@ +// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package utils + +import ( + "context" + "crypto/rand" + "encoding/binary" + "encoding/hex" + "math" + "math/big" + "reflect" + "strconv" + "strings" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/ttime" +) + +func DefaultIfBlank(str string, default_value string) string { + if len(str) == 0 { + return default_value + } + return str +} + +func ZeroOrNil(obj interface{}) bool { + value := reflect.ValueOf(obj) + if !value.IsValid() { + return true + } + if obj == nil { + return true + } + switch value.Kind() { + case reflect.Slice, reflect.Array, reflect.Map: + return value.Len() == 0 + } + zero := reflect.Zero(reflect.TypeOf(obj)) + if !value.Type().Comparable() { + return false + } + if obj == zero.Interface() { + return true + } + return false +} + +// SlicesDeepEqual checks if slice1 and slice2 are equal, disregarding order. +func SlicesDeepEqual(slice1, slice2 interface{}) bool { + s1 := reflect.ValueOf(slice1) + s2 := reflect.ValueOf(slice2) + + if s1.Len() != s2.Len() { + return false + } + if s1.Len() == 0 { + return true + } + + s2found := make([]int, s2.Len()) +OuterLoop: + for i := 0; i < s1.Len(); i++ { + s1el := s1.Slice(i, i+1) + for j := 0; j < s2.Len(); j++ { + if s2found[j] == 1 { + // We already counted this s2 element + continue + } + s2el := s2.Slice(j, j+1) + if reflect.DeepEqual(s1el.Interface(), s2el.Interface()) { + s2found[j] = 1 + continue OuterLoop + } + } + // Couldn't find something unused equal to s1 + return false + } + return true +} + +func RandHex() string { + randInt, _ := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) + out := make([]byte, 10) + binary.PutVarint(out, randInt.Int64()) + return hex.EncodeToString(out) +} + +func Strptr(s string) *string { + return &s +} + +var _time ttime.Time = &ttime.DefaultTime{} + +// RetryWithBackoff takes a Backoff and a function to call that returns an error +// If the error is nil then the function will no longer be called +// If the error is Retriable then that will be used to determine if it should be +// retried +func RetryWithBackoff(backoff Backoff, fn func() error) error { + return RetryWithBackoffCtx(context.Background(), backoff, fn) +} + +// RetryWithBackoffCtx takes a context, a Backoff, and a function to call that returns an error +// If the context is done, nil will be returned +// If the error is nil then the function will no longer be called +// If the error is Retriable then that will be used to determine if it should be +// retried +func RetryWithBackoffCtx(ctx context.Context, backoff Backoff, fn func() error) error { + var err error + for { + select { + case <-ctx.Done(): + return nil + default: + } + + err = fn() + + retriableErr, isRetriableErr := err.(Retriable) + + if err == nil || (isRetriableErr && !retriableErr.Retry()) { + return err + } + + _time.Sleep(backoff.Duration()) + } + return err +} + +// RetryNWithBackoff takes a Backoff, a maximum number of tries 'n', and a +// function that returns an error. The function is called until either it does +// not return an error or the maximum tries have been reached. +// If the error returned is Retriable, the Retriability of it will be respected. +// If the number of tries is exhausted, the last error will be returned. +func RetryNWithBackoff(backoff Backoff, n int, fn func() error) error { + return RetryNWithBackoffCtx(context.Background(), backoff, n, fn) +} + +// RetryNWithBackoffCtx takes a context, a Backoff, a maximum number of tries 'n', and a function that returns an error. +// The function is called until it does not return an error, the context is done, or the maximum tries have been +// reached. +// If the error returned is Retriable, the Retriability of it will be respected. +// If the number of tries is exhausted, the last error will be returned. +func RetryNWithBackoffCtx(ctx context.Context, backoff Backoff, n int, fn func() error) error { + var err error + RetryWithBackoffCtx(ctx, backoff, func() error { + err = fn() + n-- + if n == 0 { + // Break out after n tries + return nil + } + return err + }) + return err +} + +// Uint16SliceToStringSlice converts a slice of type uint16 to a slice of type +// *string. It uses strconv.Itoa on each element +func Uint16SliceToStringSlice(slice []uint16) []*string { + stringSlice := make([]*string, len(slice)) + for i, el := range slice { + str := strconv.Itoa(int(el)) + stringSlice[i] = &str + } + return stringSlice +} + +func StrSliceEqual(s1, s2 []string) bool { + if len(s1) != len(s2) { + return false + } + + for i := 0; i < len(s1); i++ { + if s1[i] != s2[i] { + return false + } + } + return true +} + +func ParseBool(str string, default_ bool) bool { + res, err := strconv.ParseBool(strings.TrimSpace(str)) + if err != nil { + return default_ + } + return res +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go new file mode 100644 index 0000000000..dd3595605e --- /dev/null +++ b/pkg/utils/utils_test.go @@ -0,0 +1,276 @@ +// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package utils + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/aws/amazon-ecs-agent/agent/utils/ttime" + "github.com/aws/amazon-ecs-agent/agent/utils/ttime/mocks" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func TestDefaultIfBlank(t *testing.T) { + const defaultValue = "a boring default" + const specifiedValue = "new value" + result := DefaultIfBlank(specifiedValue, defaultValue) + assert.Equal(t, specifiedValue, result) + + result = DefaultIfBlank("", defaultValue) + assert.Equal(t, defaultValue, result) +} + +func TestZeroOrNil(t *testing.T) { + type ZeroTest struct { + testInt int + TestStr string + } + + var strMap map[string]string + + testCases := []struct { + param interface{} + expected bool + name string + }{ + {nil, true, "Nil is nil"}, + {0, true, "0 is 0"}, + {"", true, "\"\" is the string zerovalue"}, + {ZeroTest{}, true, "ZeroTest zero-value should be zero"}, + {ZeroTest{TestStr: "asdf"}, false, "ZeroTest with a field populated isn't zero"}, + {1, false, "1 is not 0"}, + {[]uint16{1, 2, 3}, false, "[1,2,3] is not zero"}, + {[]uint16{}, true, "[] is zero"}, + {struct{ uncomparable []uint16 }{uncomparable: []uint16{1, 2, 3}}, false, "Uncomparable structs are never zero"}, + {struct{ uncomparable []uint16 }{uncomparable: nil}, false, "Uncomparable structs are never zero"}, + {strMap, true, "map[string]string is zero or nil"}, + {make(map[string]string), true, "empty map[string]string is zero or nil"}, + {map[string]string{"foo": "bar"}, false, "map[string]string{foo:bar} is not zero or nil"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, ZeroOrNil(tc.param), tc.name) + }) + } +} + +func TestSlicesDeepEqual(t *testing.T) { + testCases := []struct { + left []string + right []string + expected bool + name string + }{ + {[]string{}, []string{}, true, "Two empty slices"}, + {[]string{"cat"}, []string{}, false, "One empty slice"}, + {[]string{}, []string{"cat"}, false, "Another empty slice"}, + {[]string{"cat"}, []string{"cat"}, true, "Two slices with one element each"}, + {[]string{"cat", "dog", "cat"}, []string{"dog", "cat", "cat"}, true, "Two slices with multiple elements"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, SlicesDeepEqual(tc.left, tc.right)) + }) + } +} + +func TestRetryWithBackoff(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mocktime := mock_ttime.NewMockTime(ctrl) + _time = mocktime + defer func() { _time = &ttime.DefaultTime{} }() + + t.Run("retries", func(t *testing.T) { + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(3) + counter := 3 + RetryWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error { + if counter == 0 { + return nil + } + counter-- + return errors.New("err") + }) + assert.Equal(t, 0, counter, "Counter didn't go to 0; didn't get retried enough") + }) + + t.Run("no retries", func(t *testing.T) { + // no sleeps + RetryWithBackoff(NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error { + return NewRetriableError(NewRetriable(false), errors.New("can't retry")) + }) + }) +} + +func TestRetryWithBackoffCtx(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mocktime := mock_ttime.NewMockTime(ctrl) + _time = mocktime + defer func() { _time = &ttime.DefaultTime{} }() + + t.Run("retries", func(t *testing.T) { + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(3) + counter := 3 + RetryWithBackoffCtx(context.TODO(), NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error { + if counter == 0 { + return nil + } + counter-- + return errors.New("err") + }) + assert.Equal(t, 0, counter, "Counter didn't go to 0; didn't get retried enough") + }) + + t.Run("no retries", func(t *testing.T) { + // no sleeps + RetryWithBackoffCtx(context.TODO(), NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error { + return NewRetriableError(NewRetriable(false), errors.New("can't retry")) + }) + }) + + t.Run("cancel context", func(t *testing.T) { + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(2) + counter := 2 + ctx, cancel := context.WithCancel(context.TODO()) + RetryWithBackoffCtx(ctx, NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error { + counter-- + if counter == 0 { + cancel() + } + return errors.New("err") + }) + assert.Equal(t, 0, counter, "Counter not 0; went the wrong number of times") + }) + +} + +func TestRetryNWithBackoff(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mocktime := mock_ttime.NewMockTime(ctrl) + _time = mocktime + defer func() { _time = &ttime.DefaultTime{} }() + + t.Run("count exceeded", func(t *testing.T) { + // 2 tries, 1 sleep + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(1) + counter := 3 + err := RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 2, func() error { + counter-- + return errors.New("err") + }) + assert.Equal(t, 1, counter, "Should have stopped after two tries") + assert.Error(t, err) + }) + + t.Run("retry succeeded", func(t *testing.T) { + // 3 tries, 2 sleeps + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(2) + counter := 3 + err := RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error { + counter-- + if counter == 0 { + return nil + } + return errors.New("err") + }) + assert.Equal(t, 0, counter) + assert.NoError(t, err) + }) +} + +func TestRetryNWithBackoffCtx(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mocktime := mock_ttime.NewMockTime(ctrl) + _time = mocktime + defer func() { _time = &ttime.DefaultTime{} }() + + t.Run("count exceeded", func(t *testing.T) { + // 2 tries, 1 sleep + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(1) + counter := 3 + err := RetryNWithBackoffCtx(context.TODO(), NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 2, func() error { + counter-- + return errors.New("err") + }) + assert.Equal(t, 1, counter, "Should have stopped after two tries") + assert.Error(t, err) + }) + + t.Run("retry succeeded", func(t *testing.T) { + // 3 tries, 2 sleeps + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(2) + counter := 3 + err := RetryNWithBackoffCtx(context.TODO(), NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error { + counter-- + if counter == 0 { + return nil + } + return errors.New("err") + }) + assert.Equal(t, 0, counter) + assert.NoError(t, err) + }) + + t.Run("cancel context", func(t *testing.T) { + // 2 tries, 2 sleeps + mocktime.EXPECT().Sleep(100 * time.Millisecond).Times(2) + counter := 3 + ctx, cancel := context.WithCancel(context.TODO()) + err := RetryNWithBackoffCtx(ctx, NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error { + counter-- + if counter == 1 { + cancel() + } + return errors.New("err") + }) + assert.Equal(t, 1, counter, "Should have stopped after two tries") + assert.Error(t, err) + }) +} + +func TestParseBool(t *testing.T) { + truthyStrings := []string{"true", "1", "t", "true\r", "true ", "true \r"} + falsyStrings := []string{"false", "0", "f", "false\r", "false ", "false \r"} + neitherStrings := []string{"apple", " ", "\r", "orange", "maybe"} + + for _, str := range truthyStrings { + t.Run("truthy", func(t *testing.T) { + assert.True(t, ParseBool(str, false), "Truthy string should be truthy") + assert.True(t, ParseBool(str, true), "Truthy string should be truthy (regardless of default)") + }) + } + + for _, str := range falsyStrings { + t.Run("falsy", func(t *testing.T) { + assert.False(t, ParseBool(str, false), "Falsy string should be falsy") + assert.False(t, ParseBool(str, true), "Falsy string should be falsy (regardless of default)") + }) + } + + for _, str := range neitherStrings { + t.Run("defaults", func(t *testing.T) { + assert.False(t, ParseBool(str, false), "Should default to false") + assert.True(t, ParseBool(str, true), "Should default to true") + }) + } +} diff --git a/scripts/aws-cni-support.sh b/scripts/aws-cni-support.sh new file mode 100755 index 0000000000..ffcb05dfe3 --- /dev/null +++ b/scripts/aws-cni-support.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# This script generates a file in go with the license contents as a constant + +# Set language to C to make sorting consistent among different environments. +export LANG=C + +set -e +LOG_DIR="/var/log/aws-routed-eni" + +# collecting L-IPAMD introspection data +curl http://localhost:51678/v1/enis > ${LOG_DIR}/eni.output +curl http://localhost:51678/v1/pods > ${LOG_DIR}/pod.output + +# collecting kubeleet introspection data +curl http://localhost:10255/pods > ${LOG_DIR}/kubelet.output + +# ifconfig +ifconfig > ${LOG_DIR}/ifconig.output + +# ip rule show +ip rule show > ${LOG_DIR}/iprule.output + +# dump out route table +ROUTE_OUTPUT="route.output" +echo "=============================================" >> ${LOG_DIR}/${ROUTE_OUTPUT} +echo "ip route show table all" >> $LOG_DIR/$ROUTE_OUTPUT +ip route show table all >> $LOG_DIR/$ROUTE_OUTPUT + +tar -cvzf $LOG_DIR/aws-cni-support.tar.gz ${LOG_DIR}/ diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index c4a702c314..d050c8d6f0 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -14,5 +14,6 @@ COPY misc/aws.conf /app COPY misc/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ADD aws-k8s-agent /app +ADD scripts/aws-cni-support.sh /app ADD scripts/install-aws.sh /app ENTRYPOINT /app/install-aws.sh diff --git a/scripts/install-aws.sh b/scripts/install-aws.sh index 13f9fbad4d..1922e959f6 100755 --- a/scripts/install-aws.sh +++ b/scripts/install-aws.sh @@ -1,5 +1,6 @@ echo "=====Starting installing AWS-CNI =========" cp /app/aws-cni /host/opt/cni/bin/ +cp /app/aws-cni-support.sh /host/opt/cni/bin/ cp /app/aws.conf /host/etc/cni/net.d/ echo "=====Starting amazon-k8s-agent ===========" /app/aws-k8s-agent