diff --git a/test/framework/options.go b/test/framework/options.go index 4c7cf34065..b006b1c1bd 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -27,13 +27,15 @@ func init() { } type Options struct { - KubeConfig string - ClusterName string - AWSRegion string - AWSVPCID string - NgNameLabelKey string - NgNameLabelVal string - EKSEndpoint string + KubeConfig string + ClusterName string + AWSRegion string + AWSVPCID string + NgNameLabelKey string + NgNameLabelVal string + EKSEndpoint string + InitialCNIVersion string + FinalCNIVersion string } func (options *Options) BindFlags() { @@ -44,6 +46,8 @@ func (options *Options) BindFlags() { flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name") flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name") flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint") + flag.StringVar(&options.InitialCNIVersion, "initial-version", "", "Initial CNI version before upgrade applied") + flag.StringVar(&options.FinalCNIVersion, "final-version", "", "Final CNI version after upgrade applied") } func (options *Options) Validate() error { diff --git a/test/framework/resources/aws/services/eks.go b/test/framework/resources/aws/services/eks.go index 838870d9b4..ff7ada83a4 100644 --- a/test/framework/resources/aws/services/eks.go +++ b/test/framework/resources/aws/services/eks.go @@ -22,14 +22,25 @@ import ( type EKS interface { DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) - CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) + CreateAddon(createAddOnParams AddOnInput) (*eks.CreateAddonOutput, error) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) + DescribeAddonVersions(addon string, k8sVersion string) (*eks.DescribeAddonVersionsOutput, error) + DescribeAddon(addon string, clusterName string) (*eks.DescribeAddonOutput, error) } type defaultEKS struct { eksiface.EKSAPI } +// struct added to make params passing extensible +type AddOnInput struct { + _ struct{} `type:"structure"` + + AddonName string `locationName:"addonName" type:"string" required:"true"` + ClusterName string `location:"uri" locationName:"name" min:"1" type:"string" required:"true"` + AddonVersion string `locationName:"addonVersion" type:"string"` // eg format: v1.9.3-eksbuild.1 +} + func NewEKS(session *session.Session, endpoint string) EKS { return &defaultEKS{ EKSAPI: eks.New(session, &aws.Config{ @@ -39,10 +50,14 @@ func NewEKS(session *session.Session, endpoint string) EKS { } } -func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) { +func (d defaultEKS) CreateAddon(addOnInput AddOnInput) (*eks.CreateAddonOutput, error) { createAddonInput := &eks.CreateAddonInput{ - AddonName: aws.String(addon), - ClusterName: aws.String(clusterName), + AddonName: aws.String(addOnInput.AddonName), + ClusterName: aws.String(addOnInput.ClusterName), + } + if addOnInput.AddonVersion != "" { + createAddonInput.SetAddonVersion(addOnInput.AddonVersion) + createAddonInput.SetResolveConflicts("OVERWRITE") } return d.EKSAPI.CreateAddon(createAddonInput) } @@ -55,6 +70,22 @@ func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAd return d.EKSAPI.DeleteAddon(deleteAddonInput) } +func (d defaultEKS) DescribeAddonVersions(addon string, k8sVersion string) (*eks.DescribeAddonVersionsOutput, error) { + describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{ + AddonName: aws.String(addon), + KubernetesVersion: aws.String(k8sVersion), + } + return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput) +} + +func (d defaultEKS) DescribeAddon(addon string, clusterName string) (*eks.DescribeAddonOutput, error) { + describeAddonInput := &eks.DescribeAddonInput{ + AddonName: aws.String(addon), + ClusterName: aws.String(clusterName), + } + return d.EKSAPI.DescribeAddon(describeAddonInput) +} + func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) { describeClusterInput := &eks.DescribeClusterInput{ Name: aws.String(clusterName), diff --git a/test/integration-new/cni/cni_version_test.go b/test/integration-new/cni/cni_version_test.go new file mode 100644 index 0000000000..45182fa5ee --- /dev/null +++ b/test/integration-new/cni/cni_version_test.go @@ -0,0 +1,127 @@ +package cni + +import ( + "context" + "fmt" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" + + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/aws-sdk-go/service/eks" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + describeAddonOutput *eks.DescribeAddonOutput + err error + initialCNIVersion string + finalCNIVersion string +) + +var _ = Describe("test cluster upgrade/downgrade", func() { + + It("should apply initial addon version successfully", func() { + By("getting initial cni version") + if len(f.Options.InitialCNIVersion) == 0 { + err = errors.Errorf("%s must be set!", "initial-version") + } + Expect(err).ToNot(HaveOccurred()) + initialCNIVersion = f.Options.InitialCNIVersion + ApplyAddOn(initialCNIVersion) + + }) + + Context("when testing pod networking on initial version", func() { + HostNetworkingTest() + //PodTrafficTest() + //ServiceConnectivityTest() + }) + + It("should apply final addon version successfully", func() { + By("getting final cni version") + if len(f.Options.FinalCNIVersion) == 0 { + err = errors.Errorf("%s must be set!", "final-version") + } + Expect(err).ToNot(HaveOccurred()) + finalCNIVersion = f.Options.FinalCNIVersion + ApplyAddOn(finalCNIVersion) + + }) + + Context("when testing pod networking on final version", func() { + HostNetworkingTest() + //PodTrafficTest() + //ServiceConnectivityTest() + }) + +}) + +func ApplyAddOn(versionName string) { + + By("getting the current addon") + describeAddonOutput, err = f.CloudServices.EKS().DescribeAddon("vpc-cni", f.Options.ClusterName) + if err == nil { + By("checking if the current addon is same as addon to be applied") + if *describeAddonOutput.Addon.AddonVersion != versionName { + + By("deleting the current vpc cni addon ") + _, err = f.CloudServices.EKS().DeleteAddon("vpc-cni", f.Options.ClusterName) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for addon to be deleted") + err = DeleteAddOnAndWaitTillReady("vpc-cni", f.Options.ClusterName) + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("By applying addon %s\n", versionName)) + _, err = f.CloudServices.EKS().CreateAddon(services.AddOnInput{AddonName: "vpc-cni", ClusterName: f.Options.ClusterName, AddonVersion: versionName}) + Expect(err).ToNot(HaveOccurred()) + + } + } else { + By(fmt.Sprintf("By applying addon %s\n", versionName)) + _, err = f.CloudServices.EKS().CreateAddon(services.AddOnInput{AddonName: "vpc-cni", ClusterName: f.Options.ClusterName, AddonVersion: versionName}) + Expect(err).ToNot(HaveOccurred()) + } + + By("waiting for addon to be ACTIVE") + err = UpdateAddOnAndWaitTillReady("vpc-cni", f.Options.ClusterName) + Expect(err).ToNot(HaveOccurred()) + + //Set the WARM_ENI_TARGET to 0 to prevent all pods being scheduled on secondary ENI + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system", + "aws-node", map[string]string{"WARM_IP_TARGET": "3", "WARM_ENI_TARGET": "0"}) +} + +func DeleteAddOnAndWaitTillReady(addOnName string, clusterName string) error { + + ctx := context.Background() + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + _, err = f.CloudServices.EKS().DescribeAddon(addOnName, clusterName) + if err != nil { + return true, nil + } + return false, nil + }, ctx.Done()) + +} + +func UpdateAddOnAndWaitTillReady(addOnName string, clusterName string) error { + + ctx := context.Background() + var status = "" + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + describeAddonOutput, err = f.CloudServices.EKS().DescribeAddon(addOnName, clusterName) + if err != nil { + return false, err + } + status = *describeAddonOutput.Addon.Status + if status == "ACTIVE" { + return true, nil + } + return false, nil + }, ctx.Done()) +} diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 03a3abe687..f544fe4066 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -49,7 +49,10 @@ const ( DEFAULT_VETH_PREFIX = "eni" ) -var _ = Describe("test host networking", func() { +var _ = Describe("test host networking", HostNetworkingTest) + +func HostNetworkingTest() { + var err error var podLabelKey = "app" var podLabelVal = "host-networking-test" @@ -205,7 +208,7 @@ var _ = Describe("test host networking", func() { Expect(err).ToNot(HaveOccurred()) }) }) -}) +} // Validate host networking for the list of pods supplied func ValidateHostNetworking(testType TestType, podValidationInputString string) { diff --git a/test/integration-new/cni/pod_traffic_test.go b/test/integration-new/cni/pod_traffic_test.go index 7014ddac61..06f1f282b3 100644 --- a/test/integration-new/cni/pod_traffic_test.go +++ b/test/integration-new/cni/pod_traffic_test.go @@ -32,7 +32,9 @@ import ( // Verifies network connectivity across Pods placed on different combination of // primary and second Elastic Networking Interface on two nodes. The test verifies // different traffic type for instance TCP, UDP, ICMP -var _ = Describe("test pod networking", func() { +var _ = Describe("test pod networking", PodTrafficTest) + +func PodTrafficTest() { var ( err error @@ -263,7 +265,7 @@ var _ = Describe("test pod networking", func() { testFailedConnectionCommandFunc) }) }) -}) +} func VerifyConnectivityFailsForNegativeCase(senderPod coreV1.Pod, receiverPod coreV1.Pod, port int, getTestCommandFunc func(receiverPod coreV1.Pod, port int) []string) { diff --git a/test/integration-new/cni/service_connectivity_test.go b/test/integration-new/cni/service_connectivity_test.go index 94e9c94470..47ca4af233 100644 --- a/test/integration-new/cni/service_connectivity_test.go +++ b/test/integration-new/cni/service_connectivity_test.go @@ -34,7 +34,9 @@ const ( ) // Verifies connectivity to deployment behind different service types -var _ = Describe("[CANARY] test service connectivity", func() { +var _ = Describe("[CANARY] test service connectivity", ServiceConnectivityTest) + +func ServiceConnectivityTest() { var err error // Deployment running the http server @@ -177,4 +179,4 @@ var _ = Describe("[CANARY] test service connectivity", func() { }) //TODO: Add test case to install lb controller and test with nlb-ip mode -}) +} diff --git a/test/integration-new/ipamd/ipamd_suite_test.go b/test/integration-new/ipamd/ipamd_suite_test.go index cb82f3ee8d..175a580669 100644 --- a/test/integration-new/ipamd/ipamd_suite_test.go +++ b/test/integration-new/ipamd/ipamd_suite_test.go @@ -16,6 +16,8 @@ package ipamd import ( "testing" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" + "github.com/aws/amazon-vpc-cni-k8s/test/framework" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/aws-sdk-go/service/ec2" @@ -61,7 +63,7 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { if addonDeleteError == nil { By("Restore coredns addon") - _, err := f.CloudServices.EKS().CreateAddon("coredns", f.Options.ClusterName) + _, err := f.CloudServices.EKS().CreateAddon(services.AddOnInput{AddonName: "coredns", ClusterName: f.Options.ClusterName}) Expect(err).NotTo(HaveOccurred()) } })