✨ Add e2e-test (#36)

* add ci

* mv jenkinsfile to root

* fix ci

* fix ci

Jenkinsfile

@@ -0,0 +1,45 @@
+pipeline {
+  agent {
+    docker {
+      image 'kubespheredev/porter-infra:0.0.1'
+      args '-v $HOME:/root -v /var/run/docker.sock:/var/run/docker.sock  -v /usr/bin/docker:/usr/bin/docker -v /tmp:/tmp'
+    }
+  }
+  environment {
+    tag = sh(
+      script: 'git rev-parse --short HEAD',
+      returnStdout: true
+    ).trim()
+    ACCESS_KEY_ID     = credentials('jenkins-qc-secret-key-id')
+    SECRET_ACCESS_KEY = credentials('jenkins-qc-secret-access-key')
+    IMG =  "magicsong/cloud-manager:$tag"
+  }
+  stages {
+    stage('Building Manager'){
+      steps{
+        sh """
+            CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags "-w" -o bin/manager ./cmd/main.go
+            docker build -t $IMG  -f deploy/Dockerfile bin/
+            echo "Push images"
+            docker push $IMG
+        """
+      }
+    }
+    stage('Test') {
+      steps {
+        sh """
+            ./hack/e2e.sh -s
+          """
+      }
+    }
+  }
+  post {
+        always {
+            echo 'Clean images'
+            archiveArtifacts artifacts: 'test/*.yaml', fingerprint: true
+            sh """
+              docker rmi $IMG
+            """
+        }
+    }
+}

Makefile

@@ -88,11 +88,11 @@ publish                         : bin/.docker_label
 
 clean                           :
 								rm -rf bin/ && if -f bin/.docker-images-build-timestamp then docker rmi `cat bin/.docker-images-build-timestamp`
-test                            :  
+test                            : vet
 								go test -cover $(TEST_PACKAGES)
 vet:
-								go vet ./qingcloud/... ./cmd/...
+								go vet ./qingcloud/... ./cmd/... ./test/pkg/...
 
 debug:
-								./hack/e2e.sh
+								./hack/debug.sh
 .PHONY							: default all go-build clean install-docker test

go.mod

@@ -4,8 +4,11 @@ go 1.12
 
 require (
 	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/sirupsen/logrus v1.4.1 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cobra v0.0.3 // indirect
 	github.com/yunify/qingcloud-sdk-go v2.0.0-alpha.35+incompatible
 	gopkg.in/gcfg.v1 v1.2.3

hack/debug.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+SKIP_BUILD=no
+tag=`git rev-parse --short HEAD`
+IMG=magicsong/cloud-manager:$tag
+DEST=test/manager.yaml
+#build binary
+
+set -e
+
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    -s|--skip-build)
+    SKIP_BUILD=yes
+    shift # past argument
+    ;;
+    -n|--NAMESPACE)
+    TEST_NS=$2
+    shift # past argument
+    shift # past value
+    ;;
+    -t|--tag)
+    tag="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    --default)
+    DEFAULT=YES
+    shift # past argument
+    ;;
+    *)    # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift # past argument
+    ;;
+esac
+done
+
+if [ $SKIP_BUILD == "no" ]; then
+    echo "Building binary"
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-w" -o bin/manager ./cmd/main.go
+
+    echo "Building docker image"
+    docker build -t $IMG  -f deploy/Dockerfile bin/
+    echo "Push images"
+    docker push $IMG
+    echo "Generating yaml"
+
+fi
+sed -e 's@image: .*@image: '"${IMG}"'@' deploy/kube-cloud-controller-manager.yaml > $DEST
+kubectl apply -f $DEST

hack/e2e.sh

@@ -1,12 +1,27 @@
 #!/bin/bash
 
+set -e
+
+
 SKIP_BUILD=no
 tag=`git rev-parse --short HEAD`
 IMG=magicsong/cloud-manager:$tag
 DEST=test/manager.yaml
+TEST_NS=cloud-test-$tag
 #build binary
 
+function cleanup(){
+    result=$?
+    set +e
+    echo "Cleaning Namespace"
+    kubectl delete ns $TEST_NS > /dev/null
+    if [ $SKIP_BUILD == "no" ]; then
+        docker  rmi $IMG
+    fi
+    exit $result
+}
 set -e
+trap cleanup EXIT SIGINT SIGQUIT
 
 while [[ $# -gt 0 ]]
 do
@@ -49,5 +64,10 @@ if [ $SKIP_BUILD == "no" ]; then
     echo "Generating yaml"
 
 fi
-sed -e 's@image: .*@image: '"${IMG}"'@' deploy/kube-cloud-controller-manager.yaml > $DEST
-kubectl apply -f $DEST
+sed -e 's@image: .*@image: '"${IMG}"'@' -e 's/kube-system/'"$TEST_NS"'/g' deploy/kube-cloud-controller-manager.yaml > $DEST
+
+kubectl create ns $TEST_NS
+kubectl apply -f $DEST
+export TEST_NS
+
+go test -v -mod=vendor ./test/pkg/e2e/

test/pkg/e2e/e2e_suite_test.go

@@ -0,0 +1,71 @@
+package e2e_test
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/yunify/qingcloud-cloud-controller-manager/test/pkg/e2eutil"
+	qc "github.com/yunify/qingcloud-sdk-go/service"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func TestE2e(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "E2e Suite")
+}
+
+const ControllerName = "cloud-controller-manager"
+
+var (
+	workspace     string
+	testNamespace string
+	k8sclient     *kubernetes.Clientset
+	qcService     *qc.QingCloudService
+)
+
+func getWorkspace() string {
+	_, filename, _, _ := runtime.Caller(0)
+	return path.Dir(filename)
+}
+
+var _ = BeforeSuite(func() {
+	//init qcservice
+	qcs, err := e2eutil.GetQingcloudService()
+	Expect(err).ShouldNot(HaveOccurred(), "Failed init qc service")
+	qcService = qcs
+	testNamespace = os.Getenv("TEST_NS")
+	Expect(testNamespace).ShouldNot(BeEmpty())
+	workspace = getWorkspace() + "/../../.."
+	home := homeDir()
+	Expect(home).ShouldNot(BeEmpty())
+	//read config
+	c, err := clientcmd.BuildConfigFromFlags("", filepath.Join(home, ".kube", "config"))
+	Expect(err).ShouldNot(HaveOccurred(), "Error in load kubeconfig")
+	k8sclient = kubernetes.NewForConfigOrDie(c)
+
+	//kubectl apply
+	err = e2eutil.WaitForController(k8sclient, testNamespace, ControllerName, time.Second*10, time.Minute*2)
+	Expect(err).ShouldNot(HaveOccurred(), "Failed to start controller")
+	log.Println("Ready for testing")
+})
+
+var _ = AfterSuite(func() {
+	cmd := exec.Command("kubectl", "delete", "-f", workspace+"/test/manager.yaml")
+	Expect(cmd.Run()).ShouldNot(HaveOccurred())
+})
+
+func homeDir() string {
+	if h := os.Getenv("HOME"); h != "" {
+		return h
+	}
+	return ""
+}

test/pkg/e2e/e2e_test.go

@@ -0,0 +1,68 @@
+package e2e_test
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/yunify/qingcloud-cloud-controller-manager/test/pkg/e2eutil"
+	"github.com/yunify/qingcloud-sdk-go/service"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("E2e", func() {
+	It("Should work as expected when using sample yamls", func() {
+		//apply service
+		service1Path := workspace + "/test/samples/service.yaml"
+		serviceName := "mylbapp"
+		testip := "139.198.121.161"
+		Expect(e2eutil.KubectlApply(service1Path)).ShouldNot(HaveOccurred())
+		defer func() {
+			Expect(e2eutil.KubectlDelete(service1Path)).ShouldNot(HaveOccurred())
+			//make sure lb is deleted
+			lbService, _ := qcService.LoadBalancer("ap2a")
+			Eventually(func() error {
+				unaccept1 := "pending"
+				unaccept2 := "active"
+				key := "k8s_mylbapp"
+				input := &service.DescribeLoadBalancersInput{
+					Status:     []*string{&unaccept1, &unaccept2},
+					SearchWord: &key,
+				}
+				output, err := lbService.DescribeLoadBalancers(input)
+				if err != nil {
+					return err
+				}
+				if len(output.LoadBalancerSet) == 0 {
+					return nil
+				}
+				log.Printf("id:%s, name:%s, status:%s", *output.LoadBalancerSet[0].LoadBalancerID, *output.LoadBalancerSet[0].LoadBalancerName, *output.LoadBalancerSet[0].Status)
+				return fmt.Errorf("LB has not been deleted")
+			}, time.Minute*2, time.Second*10).Should(Succeed())
+		}()
+		Eventually(func() error {
+			service, err := k8sclient.CoreV1().Services("default").Get(serviceName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP == testip {
+				return nil
+			}
+			return fmt.Errorf("Still got no ip")
+		}, 3*time.Minute, 20*time.Second).Should(Succeed())
+		log.Println("Successfully assign a ip")
+		url := fmt.Sprintf("http://%s:%d", testip, 8088)
+		Eventually(func() int {
+			resp, err := http.Get(url)
+			if err != nil {
+				log.Println("Error in sending request,err: " + err.Error())
+				return -1
+			}
+			return resp.StatusCode
+		}, time.Second*20, time.Minute*5).Should(Equal(http.StatusOK))
+		log.Println("Successfully get a 200 response")
+	})
+})

test/pkg/e2eutil/e2eutil.go

@@ -0,0 +1,58 @@
+package e2eutil
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/yunify/qingcloud-sdk-go/config"
+	qc "github.com/yunify/qingcloud-sdk-go/service"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+)
+
+func WaitForController(c *kubernetes.Clientset, namespace, name string, retryInterval, timeout time.Duration) error {
+	err := wait.Poll(retryInterval, timeout, func() (done bool, err error) {
+		controller, err := c.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{})
+		if apierrors.IsNotFound(err) {
+			fmt.Println("Cannot find controller")
+			return false, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		if controller.Status.ReadyReplicas == 1 {
+			return true, nil
+		}
+		return false, nil
+	})
+	return err
+}
+
+func KubectlApply(filename string) error {
+	cmd := exec.Command("kubectl", "apply", "-f", filename)
+	str, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Printf("kubectl apply failed, error :%s\n", str)
+	}
+	return err
+}
+
+func KubectlDelete(filename string) error {
+	ctx, cancle := context.WithTimeout(context.Background(), time.Second*20)
+	cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", filename)
+	defer cancle()
+	return cmd.Run()
+}
+
+func GetQingcloudService() (*qc.QingCloudService, error) {
+	accessKey := os.Getenv("ACCESS_KEY_ID")
+	secret := os.Getenv("SECRET_ACCESS_KEY")
+	configuration, _ := config.New(accessKey, secret)
+	configuration.Zone = "ap2a"
+	return qc.Init(configuration)
+}

test/pkg/e2eutil/http.go

@@ -0,0 +1 @@
+package e2eutil

test/sample/service.yaml → test/samples/service.yaml

@@ -3,7 +3,6 @@ apiVersion: v1
 metadata:
   name:  mylbapp
   annotations:
-    lb.kubesphere.io/v1alpha1: porter
     service.beta.kubernetes.io/qingcloud-load-balancer-eip-ids: "eip-vmldumvv"
     service.beta.kubernetes.io/qingcloud-load-balancer-type: "0"
 spec: