fix flaky canary test (aws#1742)

haouc · Feb 9, 2022 · 789582b · 789582b
1 parent c694755
commit 789582b
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 9 deletions.
diff --git a/test/framework/resources/k8s/resources/service.go b/test/framework/resources/k8s/resources/service.go
@@ -20,6 +20,7 @@ import (
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -28,7 +29,7 @@ import (
 type ServiceManager interface {
 	GetService(ctx context.Context, namespace string, name string) (*v1.Service, error)
 	CreateService(ctx context.Context, service *v1.Service) (*v1.Service, error)
-	DeleteService(ctx context.Context, service *v1.Service) error
+	DeleteAndWaitTillServiceDeleted(ctx context.Context, service *v1.Service) error
 }
 
 type defaultServiceManager struct {
@@ -69,11 +70,20 @@ func (s *defaultServiceManager) CreateService(ctx context.Context, service *v1.S
 	}, ctx.Done())
 }
 
-func (s *defaultServiceManager) DeleteService(ctx context.Context, service *v1.Service) error {
+func (s *defaultServiceManager) DeleteAndWaitTillServiceDeleted(ctx context.Context, service *v1.Service) error {
 	err := s.k8sClient.Delete(ctx, service)
 	if err != nil {
 		return err
 	}
 
-	return nil
+	observed := &v1.Service{}
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := s.k8sClient.Get(ctx, utils.NamespacedName(service), observed); err != nil {
+			if errors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
 }
diff --git a/test/integration-new/cni/service_connectivity_test.go b/test/integration-new/cni/service_connectivity_test.go
@@ -58,9 +58,12 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 
 	JustBeforeEach(func() {
 		deploymentContainer = manifest.NewBusyBoxContainerBuilder().
-			Image("python").
-			Command([]string{"python3"}).
-			Args([]string{"-m", "http.server", "80"}).Build()
+			Image("nginx:1.21.4").
+			Command(nil).
+			Port(v1.ContainerPort{
+				ContainerPort: 80,
+				Protocol:      "TCP",
+			}).Build()
 
 		deployment = manifest.NewDefaultDeploymentBuilder().
 			Name("http-server").
@@ -93,7 +96,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 
 		testerContainer = manifest.NewBusyBoxContainerBuilder().
 			Command([]string{"wget"}).
-			Args([]string{"--spider", "-T", "1", fmt.Sprintf("%s:%d", service.Spec.ClusterIP,
+			Args([]string{"--spider", "-T", "5", fmt.Sprintf("%s:%d", service.Spec.ClusterIP,
 				service.Spec.Ports[0].Port)}).
 			Build()
 
@@ -132,7 +135,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 		err = f.K8sResourceManagers.JobManager().DeleteAndWaitTillJobIsDeleted(negativeTesterJob)
 		Expect(err).ToNot(HaveOccurred())
 
-		err = f.K8sResourceManagers.ServiceManager().DeleteService(context.Background(), service)
+		err = f.K8sResourceManagers.ServiceManager().DeleteAndWaitTillServiceDeleted(context.Background(), service)
 		Expect(err).ToNot(HaveOccurred())
 
 		err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment)

diff --git a/test/integration-new/ipv6/ipv6_service_connectivity_test.go b/test/integration-new/ipv6/ipv6_service_connectivity_test.go
@@ -136,7 +136,7 @@ var _ = Describe("test service connectivity", func() {
 		err = f.K8sResourceManagers.JobManager().DeleteAndWaitTillJobIsDeleted(negativeTesterJob)
 		Expect(err).ToNot(HaveOccurred())
 
-		err = f.K8sResourceManagers.ServiceManager().DeleteService(context.Background(), service)
+		err = f.K8sResourceManagers.ServiceManager().DeleteAndWaitTillServiceDeleted(context.Background(), service)
 		Expect(err).ToNot(HaveOccurred())
 
 		err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment)