From 78f0ffd456a3452eb5353ee2150395b0e8ae8ae4 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 27 Dec 2021 10:51:15 -0800 Subject: [PATCH 1/2] e2e: Use Helm chart from the commit --- scripts/ci_e2e_test.sh | 3 +++ test/e2e/ingress/multi_path_backend_test.go | 6 ++++-- test/e2e/ingress/vanilla_ingress_test.go | 8 ++++--- test/e2e/run-canary-test.sh | 9 ++------ test/framework/controller/constants.go | 1 - .../controller/installation_manager.go | 12 +++++++---- test/framework/framework.go | 2 +- test/framework/helm/release_manager.go | 21 +++++++++---------- test/framework/options.go | 7 ++++++- 9 files changed, 39 insertions(+), 30 deletions(-) diff --git a/scripts/ci_e2e_test.sh b/scripts/ci_e2e_test.sh index b2b2fe1fe..20635d03d 100755 --- a/scripts/ci_e2e_test.sh +++ b/scripts/ci_e2e_test.sh @@ -36,6 +36,8 @@ CLUSTER_INSTANCE_TYPE="m5.xlarge" CLUSTER_NODE_COUNT="4" CLUSTER_KUBECONFIG=${CLUSTER_KUBECONFIG:-"/tmp/lb-controller-e2e/clusters/${CLUSTER_NAME}.kubeconfig"} +HELM_DIR="$(cd $(dirname "${BASH_SOURCE[0]}")/../helm ; pwd)" + ####################################### # Build and push ECR image for AWS Load Balancer Controller # @@ -227,6 +229,7 @@ test_controller_image() { --cluster-name=${CLUSTER_NAME} \ --aws-region=${AWS_REGION} \ --aws-vpc-id=${cluster_vpc_id} \ + --helm-dir=${HELM_DIR} \ --controller-image=${CONTROLLER_IMAGE_NAME} \ --s3-bucket-name=${S3_BUCKET} \ --certificate-arns=${CERTIFICATE_ARNS} diff --git a/test/e2e/ingress/multi_path_backend_test.go b/test/e2e/ingress/multi_path_backend_test.go index d711a7c4c..4708df877 100644 --- a/test/e2e/ingress/multi_path_backend_test.go +++ b/test/e2e/ingress/multi_path_backend_test.go @@ -3,6 +3,8 @@ package ingress import ( "context" "fmt" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" @@ -10,7 +12,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/test/framework" "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" - "time" ) var _ = Describe("test ingresses with multiple path and backends", func() { @@ -23,7 +24,8 @@ var _ = Describe("test ingresses with multiple path and backends", func() { if tf.Options.ControllerImage != "" { By(fmt.Sprintf("ensure cluster installed with controller: %s", tf.Options.ControllerImage), func() { - tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage) + err := tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage) + Expect(err).NotTo(HaveOccurred()) time.Sleep(60 * time.Second) }) } diff --git a/test/e2e/ingress/vanilla_ingress_test.go b/test/e2e/ingress/vanilla_ingress_test.go index f724b3adf..9b88fbd9b 100644 --- a/test/e2e/ingress/vanilla_ingress_test.go +++ b/test/e2e/ingress/vanilla_ingress_test.go @@ -3,6 +3,9 @@ package ingress import ( "context" "fmt" + "net/http" + "time" + awssdk "github.com/aws/aws-sdk-go/aws" "github.com/gavv/httpexpect/v2" . "github.com/onsi/ginkgo" @@ -12,13 +15,11 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "net/http" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/test/framework" "sigs.k8s.io/aws-load-balancer-controller/test/framework/fixture" "sigs.k8s.io/aws-load-balancer-controller/test/framework/manifest" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" - "time" ) var _ = Describe("vanilla ingress tests", func() { @@ -32,7 +33,8 @@ var _ = Describe("vanilla ingress tests", func() { ctx = context.Background() if tf.Options.ControllerImage != "" { By(fmt.Sprintf("ensure cluster installed with controller: %s", tf.Options.ControllerImage), func() { - tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage) + err := tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage) + Expect(err).NotTo(HaveOccurred()) time.Sleep(60 * time.Second) }) } diff --git a/test/e2e/run-canary-test.sh b/test/e2e/run-canary-test.sh index e8c0e4f92..f3e990471 100644 --- a/test/e2e/run-canary-test.sh +++ b/test/e2e/run-canary-test.sh @@ -62,21 +62,16 @@ eksctl create iamserviceaccount \ --override-existing-serviceaccounts \ --approve || true -echo "Update helm repo eks" -helm repo add eks https://aws.github.io/eks-charts - -helm repo update - echo "Install TargetGroupBinding CRDs" kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" echo "Install aws-load-balancer-controller" -helm upgrade -i aws-load-balancer-controller eks/aws-load-balancer-controller -n kube-system --set clusterName=$CLUSTER_NAME --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller +helm upgrade -i aws-load-balancer-controller $SCRIPT_DIR/../helm/aws-load-balancer-controller -n kube-system --set clusterName=$CLUSTER_NAME --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller #Start the test echo "Starting the ginkgo test suite" -(cd $SCRIPT_DIR && CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo -v -r --timeout 60m --failOnPending -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID || true) +(cd $SCRIPT_DIR && CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo -v -r --timeout 60m --failOnPending -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID --helm-dir=$SCRIPT_DIR/../helm || true) echo "Delete aws-load-balancer-controller" helm delete aws-load-balancer-controller -n kube-system --timeout=10m || true diff --git a/test/framework/controller/constants.go b/test/framework/controller/constants.go index d85f8b307..365624001 100644 --- a/test/framework/controller/constants.go +++ b/test/framework/controller/constants.go @@ -3,7 +3,6 @@ package controller import "time" const ( - EKSHelmChartsRepo = "https://aws.github.io/eks-charts" AWSLoadBalancerControllerHelmChart = "aws-load-balancer-controller" AWSLoadBalancerControllerHelmRelease = "aws-load-balancer-controller" AWSLoadBalancerControllerInstallationTimeout = 2 * time.Minute diff --git a/test/framework/controller/installation_manager.go b/test/framework/controller/installation_manager.go index 11711dcc5..c364d9643 100644 --- a/test/framework/controller/installation_manager.go +++ b/test/framework/controller/installation_manager.go @@ -1,11 +1,13 @@ package controller import ( + "path" + "strings" + "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/aws-load-balancer-controller/test/framework/helm" - "strings" ) // InstallationManager is responsible for manage controller installation in cluster. @@ -15,12 +17,13 @@ type InstallationManager interface { } // NewDefaultInstallationManager constructs new defaultInstallationManager. -func NewDefaultInstallationManager(helmReleaseManager helm.ReleaseManager, clusterName string, region string, vpcID string, logger logr.Logger) *defaultInstallationManager { +func NewDefaultInstallationManager(helmReleaseManager helm.ReleaseManager, clusterName string, region string, vpcID string, helmDir string, logger logr.Logger) *defaultInstallationManager { return &defaultInstallationManager{ helmReleaseManager: helmReleaseManager, clusterName: clusterName, region: region, vpcID: vpcID, + helmDir: helmDir, namespace: "kube-system", controllerSAName: "aws-load-balancer-controller", @@ -36,6 +39,7 @@ type defaultInstallationManager struct { clusterName string region string vpcID string + helmDir string namespace string controllerSAName string @@ -44,7 +48,7 @@ type defaultInstallationManager struct { func (m *defaultInstallationManager) ResetController() error { vals := m.computeDefaultHelmVals() - _, err := m.helmReleaseManager.InstallOrUpgradeRelease(EKSHelmChartsRepo, AWSLoadBalancerControllerHelmChart, + _, err := m.helmReleaseManager.InstallOrUpgradeRelease(path.Join(m.helmDir, AWSLoadBalancerControllerHelmChart), m.namespace, AWSLoadBalancerControllerHelmRelease, vals, helm.WithTimeout(AWSLoadBalancerControllerInstallationTimeout)) return err @@ -63,7 +67,7 @@ func (m *defaultInstallationManager) UpgradeController(controllerImage string) e vals["podLabels"] = map[string]string{ "revision": string(uuid.NewUUID()), } - _, err = m.helmReleaseManager.InstallOrUpgradeRelease(EKSHelmChartsRepo, AWSLoadBalancerControllerHelmChart, + _, err = m.helmReleaseManager.InstallOrUpgradeRelease(path.Join(m.helmDir, AWSLoadBalancerControllerHelmChart), m.namespace, AWSLoadBalancerControllerHelmRelease, vals, helm.WithTimeout(AWSLoadBalancerControllerInstallationTimeout)) return err diff --git a/test/framework/framework.go b/test/framework/framework.go index 50d63070a..0d225370e 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -89,6 +89,6 @@ func InitFramework() (*Framework, error) { func buildControllerInstallationManager(options Options, logger logr.Logger) controller.InstallationManager { helmReleaseManager := helm.NewDefaultReleaseManager(options.KubeConfig, logger) - ctrlInstallationManager := controller.NewDefaultInstallationManager(helmReleaseManager, options.ClusterName, options.AWSRegion, options.AWSVPCID, logger) + ctrlInstallationManager := controller.NewDefaultInstallationManager(helmReleaseManager, options.ClusterName, options.AWSRegion, options.AWSVPCID, options.HelmDir, logger) return ctrlInstallationManager } diff --git a/test/framework/helm/release_manager.go b/test/framework/helm/release_manager.go index 67b3fbb25..9cf764448 100644 --- a/test/framework/helm/release_manager.go +++ b/test/framework/helm/release_manager.go @@ -2,6 +2,8 @@ package helm import ( "fmt" + "time" + "github.com/go-logr/logr" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" @@ -9,7 +11,6 @@ import ( "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/storage/driver" "k8s.io/cli-runtime/pkg/genericclioptions" - "time" ) // ActionOptions contains general helm action options @@ -38,15 +39,15 @@ func WithTimeout(timeout time.Duration) ActionOption { // ReleaseManager is responsible for manage helm releases type ReleaseManager interface { // InstallOrUpgradeRelease install or upgrade helm release - InstallOrUpgradeRelease(chartRepo string, chartName string, + InstallOrUpgradeRelease(chartName string, namespace string, releaseName string, vals map[string]interface{}, opts ...ActionOption) (*release.Release, error) // InstallRelease install helm release - InstallRelease(chartRepo string, chartName string, + InstallRelease(chartName string, namespace string, releaseName string, vals map[string]interface{}, opts ...ActionOption) (*release.Release, error) // UpgradeRelease upgrade helm release - UpgradeRelease(chartRepo string, chartName string, + UpgradeRelease(chartName string, namespace string, releaseName string, vals map[string]interface{}, opts ...ActionOption) (*release.Release, error) } @@ -66,24 +67,23 @@ type defaultReleaseManager struct { logger logr.Logger } -func (m *defaultReleaseManager) InstallOrUpgradeRelease(chartRepo string, chartName string, +func (m *defaultReleaseManager) InstallOrUpgradeRelease(chartName string, namespace string, releaseName string, vals map[string]interface{}, opts ...ActionOption) (*release.Release, error) { actionCFG := m.obtainActionConfig(namespace) historyAction := action.NewHistory(actionCFG) historyAction.Max = 1 if _, err := historyAction.Run(releaseName); err == driver.ErrReleaseNotFound { - return m.InstallRelease(chartRepo, chartName, namespace, releaseName, vals, opts...) + return m.InstallRelease(chartName, namespace, releaseName, vals, opts...) } else { - return m.UpgradeRelease(chartRepo, chartName, namespace, releaseName, vals, opts...) + return m.UpgradeRelease(chartName, namespace, releaseName, vals, opts...) } } -func (m *defaultReleaseManager) InstallRelease(chartRepo string, chartName string, +func (m *defaultReleaseManager) InstallRelease(chartName string, namespace string, releaseName string, vals map[string]interface{}, opts ...ActionOption) (*release.Release, error) { actionCFG := m.obtainActionConfig(namespace) installAction := action.NewInstall(actionCFG) - installAction.ChartPathOptions.RepoURL = chartRepo installAction.Namespace = namespace installAction.SkipCRDs = false installAction.ReleaseName = releaseName @@ -107,12 +107,11 @@ func (m *defaultReleaseManager) InstallRelease(chartRepo string, chartName strin return installAction.Run(chartRequested, vals) } -func (m *defaultReleaseManager) UpgradeRelease(chartRepo string, chartName string, +func (m *defaultReleaseManager) UpgradeRelease(chartName string, namespace string, releaseName string, vals map[string]interface{}, opts ...ActionOption) (*release.Release, error) { actionCFG := m.obtainActionConfig(namespace) upgradeAction := action.NewUpgrade(actionCFG) - upgradeAction.ChartPathOptions.RepoURL = chartRepo upgradeAction.Namespace = namespace upgradeAction.ResetValues = true diff --git a/test/framework/options.go b/test/framework/options.go index ecbc816a7..a2bd35b6e 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -12,11 +12,12 @@ func init() { globalOptions.BindFlags() } -// configuration options +// Options are the configuration options. type Options struct { ClusterName string AWSRegion string AWSVPCID string + HelmDir string KubeConfig string // AWS Load Balancer Controller image. leave empty to use default one from helm chart. @@ -31,6 +32,7 @@ func (options *Options) BindFlags() { flag.StringVar(&options.ClusterName, "cluster-name", "", `Kubernetes cluster name (required)`) flag.StringVar(&options.AWSRegion, "aws-region", "", `AWS Region for the kubernetes cluster`) flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `ID of VPC to create load balancers in`) + flag.StringVar(&options.HelmDir, "helm-dir", "", `directory containing Helm chart`) flag.StringVar(&options.ControllerImage, "controller-image", "", `AWS Load Balancer Controller image`) @@ -51,6 +53,9 @@ func (options *Options) Validate() error { if len(options.AWSVPCID) == 0 { return errors.Errorf("%s must be set!", "aws-vpc-id") } + if len(options.HelmDir) == 0 { + return errors.Errorf("%s must be set!", "helm-dir") + } return nil } From edf56cadd5ee88707685e7c2c90f6d684b6da672 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Tue, 4 Jan 2022 16:59:11 -0800 Subject: [PATCH 2/2] Address review comments --- scripts/ci_e2e_test.sh | 2 +- test/e2e/run-canary-test.sh | 9 +++++++-- test/framework/controller/installation_manager.go | 11 +++++------ test/framework/framework.go | 2 +- test/framework/options.go | 8 +++----- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/ci_e2e_test.sh b/scripts/ci_e2e_test.sh index 20635d03d..ea6b3d078 100755 --- a/scripts/ci_e2e_test.sh +++ b/scripts/ci_e2e_test.sh @@ -229,7 +229,7 @@ test_controller_image() { --cluster-name=${CLUSTER_NAME} \ --aws-region=${AWS_REGION} \ --aws-vpc-id=${cluster_vpc_id} \ - --helm-dir=${HELM_DIR} \ + --helm-chart=${HELM_DIR}/aws-load-balancer-controller \ --controller-image=${CONTROLLER_IMAGE_NAME} \ --s3-bucket-name=${S3_BUCKET} \ --certificate-arns=${CERTIFICATE_ARNS} diff --git a/test/e2e/run-canary-test.sh b/test/e2e/run-canary-test.sh index f3e990471..e8c0e4f92 100644 --- a/test/e2e/run-canary-test.sh +++ b/test/e2e/run-canary-test.sh @@ -62,16 +62,21 @@ eksctl create iamserviceaccount \ --override-existing-serviceaccounts \ --approve || true +echo "Update helm repo eks" +helm repo add eks https://aws.github.io/eks-charts + +helm repo update + echo "Install TargetGroupBinding CRDs" kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" echo "Install aws-load-balancer-controller" -helm upgrade -i aws-load-balancer-controller $SCRIPT_DIR/../helm/aws-load-balancer-controller -n kube-system --set clusterName=$CLUSTER_NAME --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller +helm upgrade -i aws-load-balancer-controller eks/aws-load-balancer-controller -n kube-system --set clusterName=$CLUSTER_NAME --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller #Start the test echo "Starting the ginkgo test suite" -(cd $SCRIPT_DIR && CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo -v -r --timeout 60m --failOnPending -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID --helm-dir=$SCRIPT_DIR/../helm || true) +(cd $SCRIPT_DIR && CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo -v -r --timeout 60m --failOnPending -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID || true) echo "Delete aws-load-balancer-controller" helm delete aws-load-balancer-controller -n kube-system --timeout=10m || true diff --git a/test/framework/controller/installation_manager.go b/test/framework/controller/installation_manager.go index c364d9643..302158500 100644 --- a/test/framework/controller/installation_manager.go +++ b/test/framework/controller/installation_manager.go @@ -1,7 +1,6 @@ package controller import ( - "path" "strings" "github.com/go-logr/logr" @@ -17,13 +16,13 @@ type InstallationManager interface { } // NewDefaultInstallationManager constructs new defaultInstallationManager. -func NewDefaultInstallationManager(helmReleaseManager helm.ReleaseManager, clusterName string, region string, vpcID string, helmDir string, logger logr.Logger) *defaultInstallationManager { +func NewDefaultInstallationManager(helmReleaseManager helm.ReleaseManager, clusterName string, region string, vpcID string, helmChart string, logger logr.Logger) *defaultInstallationManager { return &defaultInstallationManager{ helmReleaseManager: helmReleaseManager, clusterName: clusterName, region: region, vpcID: vpcID, - helmDir: helmDir, + helmChart: helmChart, namespace: "kube-system", controllerSAName: "aws-load-balancer-controller", @@ -39,7 +38,7 @@ type defaultInstallationManager struct { clusterName string region string vpcID string - helmDir string + helmChart string namespace string controllerSAName string @@ -48,7 +47,7 @@ type defaultInstallationManager struct { func (m *defaultInstallationManager) ResetController() error { vals := m.computeDefaultHelmVals() - _, err := m.helmReleaseManager.InstallOrUpgradeRelease(path.Join(m.helmDir, AWSLoadBalancerControllerHelmChart), + _, err := m.helmReleaseManager.InstallOrUpgradeRelease(m.helmChart, m.namespace, AWSLoadBalancerControllerHelmRelease, vals, helm.WithTimeout(AWSLoadBalancerControllerInstallationTimeout)) return err @@ -67,7 +66,7 @@ func (m *defaultInstallationManager) UpgradeController(controllerImage string) e vals["podLabels"] = map[string]string{ "revision": string(uuid.NewUUID()), } - _, err = m.helmReleaseManager.InstallOrUpgradeRelease(path.Join(m.helmDir, AWSLoadBalancerControllerHelmChart), + _, err = m.helmReleaseManager.InstallOrUpgradeRelease(m.helmChart, m.namespace, AWSLoadBalancerControllerHelmRelease, vals, helm.WithTimeout(AWSLoadBalancerControllerInstallationTimeout)) return err diff --git a/test/framework/framework.go b/test/framework/framework.go index 0d225370e..7ef83ed2c 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -89,6 +89,6 @@ func InitFramework() (*Framework, error) { func buildControllerInstallationManager(options Options, logger logr.Logger) controller.InstallationManager { helmReleaseManager := helm.NewDefaultReleaseManager(options.KubeConfig, logger) - ctrlInstallationManager := controller.NewDefaultInstallationManager(helmReleaseManager, options.ClusterName, options.AWSRegion, options.AWSVPCID, options.HelmDir, logger) + ctrlInstallationManager := controller.NewDefaultInstallationManager(helmReleaseManager, options.ClusterName, options.AWSRegion, options.AWSVPCID, options.HelmChart, logger) return ctrlInstallationManager } diff --git a/test/framework/options.go b/test/framework/options.go index a2bd35b6e..7660b47e1 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -4,6 +4,7 @@ import ( "flag" "github.com/pkg/errors" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/controller" ) var globalOptions Options @@ -17,7 +18,7 @@ type Options struct { ClusterName string AWSRegion string AWSVPCID string - HelmDir string + HelmChart string KubeConfig string // AWS Load Balancer Controller image. leave empty to use default one from helm chart. @@ -32,7 +33,7 @@ func (options *Options) BindFlags() { flag.StringVar(&options.ClusterName, "cluster-name", "", `Kubernetes cluster name (required)`) flag.StringVar(&options.AWSRegion, "aws-region", "", `AWS Region for the kubernetes cluster`) flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `ID of VPC to create load balancers in`) - flag.StringVar(&options.HelmDir, "helm-dir", "", `directory containing Helm chart`) + flag.StringVar(&options.HelmChart, "helm-chart", controller.AWSLoadBalancerControllerHelmChart, `Helm chart`) flag.StringVar(&options.ControllerImage, "controller-image", "", `AWS Load Balancer Controller image`) @@ -53,9 +54,6 @@ func (options *Options) Validate() error { if len(options.AWSVPCID) == 0 { return errors.Errorf("%s must be set!", "aws-vpc-id") } - if len(options.HelmDir) == 0 { - return errors.Errorf("%s must be set!", "helm-dir") - } return nil }