From 43e09d938a1a7dd4ca4780dde6109f9390b7061d Mon Sep 17 00:00:00 2001 From: yiyiyimu Date: Tue, 26 Jan 2021 09:29:07 +0800 Subject: [PATCH] fix test Signed-off-by: yiyiyimu --- .github/workflows/chaos.yml | 35 +++++++------------ kubernetes/deployment.yaml | 2 +- kubernetes/service.yaml | 2 -- t/chaos/go.mod | 5 ++- t/chaos/go.sum | 6 ++++ t/chaos/kill-etcd.yaml | 4 +-- t/chaos/kill-etcd_test.go | 70 +++++++++++++++++++++++++++++++++---- t/chaos/utils.sh | 21 ++++++++--- 8 files changed, 106 insertions(+), 39 deletions(-) mode change 100644 => 100755 t/chaos/utils.sh diff --git a/.github/workflows/chaos.yml b/.github/workflows/chaos.yml index dec2cc45c299..1b317308bbb4 100644 --- a/.github/workflows/chaos.yml +++ b/.github/workflows/chaos.yml @@ -1,6 +1,6 @@ name: Chaos Test -on: [push, pull_request] +on: [pull_request] jobs: build: @@ -14,7 +14,7 @@ jobs: - name: Creating minikube cluster run: | - bash ./t/chaos/utils.sh startMinikube + bash ./t/chaos/utils.sh start_minikube - name: Print cluster information run: | @@ -28,29 +28,20 @@ jobs: - name: Deploy Etcd Operator run: | git clone https://github.com/yiyiyimu/etcd-operator.git --depth 1 - cd etcd-operator - bash example/rbac/create_role.sh - kubectl create -f example/deployment.yaml - bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True" - kubectl create -f example/example-etcd-cluster.yaml - bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True" + bash etcd-operator/example/rbac/create_role.sh + kubectl create -f etcd-operator/example/deployment.yaml + bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True" 30 + kubectl create -f etcd-operator/example/example-etcd-cluster.yaml + bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True" 30 - - uses: actions/checkout@v2 - name: Deploy APISIX run: | - DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}') - echo " - dns_resolver: - - ${DNS_IP} - etcd: - host: - - "http://etcd-cluster-client.default.svc.cluster.local:2379" - " > conf/config.yaml - example/rbac/create_role.sh - kubectl create configmap apisix-gw-config.yaml --from-file=../conf/config.yaml - kubectl apply -f deployment.yaml - kubectl apply -f service.yaml - while [[ $(kubectl get pods -l app=apisix-gw -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True True" ]]; do echo "Waiting for pod running" && sleep 10; done + bash ./t/chaos/utils.sh modify_config + kubectl create configmap apisix-gw-config.yaml --from-file=./conf/config.yaml + kubectl apply -f ./kubernetes/deployment.yaml + kubectl apply -f ./kubernetes/service.yaml + bash ./t/chaos/utils.sh ensure_pods_ready apisix-gw "True" 30 + nohup kubectl port-forward svc/apisix-gw-lb 9080:9080 >/dev/null 2>&1 & - name: Deploy Chaos mesh run: | diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 20cc3dc97483..cb4dcdc539eb 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -23,7 +23,7 @@ metadata: name: apisix-gw-deployment # namespace: default spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: apisix-gw diff --git a/kubernetes/service.yaml b/kubernetes/service.yaml index 401ae900ad4b..59cd8298ead1 100644 --- a/kubernetes/service.yaml +++ b/kubernetes/service.yaml @@ -41,5 +41,3 @@ spec: type: NodePort externalTrafficPolicy: Local # sessionAffinity: None - externalIPs: - - 10.253.0.11 diff --git a/t/chaos/go.mod b/t/chaos/go.mod index c5ae1378dc30..b24abf3454e3 100644 --- a/t/chaos/go.mod +++ b/t/chaos/go.mod @@ -1,5 +1,8 @@ module github.com/apache/apisix/t/chaos -require github.com/gavv/httpexpect/v2 v2.1.0 +require ( + github.com/gavv/httpexpect/v2 v2.1.0 + github.com/onsi/gomega v1.9.0 +) go 1.14 diff --git a/t/chaos/go.sum b/t/chaos/go.sum index 625a1922a1e0..5d88022bf1db 100644 --- a/t/chaos/go.sum +++ b/t/chaos/go.sum @@ -36,6 +36,8 @@ github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= @@ -78,6 +80,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpbl golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -87,5 +91,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA= diff --git a/t/chaos/kill-etcd.yaml b/t/chaos/kill-etcd.yaml index 29274d887373..98c41f6fe855 100644 --- a/t/chaos/kill-etcd.yaml +++ b/t/chaos/kill-etcd.yaml @@ -19,11 +19,11 @@ apiVersion: chaos-mesh.org/v1alpha1 kind: PodChaos metadata: name: kill-etcd - namespace: chaos-testing spec: action: pod-kill mode: all - value: "" selector: labelSelectors: "app": "etcd" + scheduler: + cron: "@every 10m" diff --git a/t/chaos/kill-etcd_test.go b/t/chaos/kill-etcd_test.go index 9b4ecbc05dcf..f14496c81558 100644 --- a/t/chaos/kill-etcd_test.go +++ b/t/chaos/kill-etcd_test.go @@ -18,16 +18,20 @@ package chaos import ( + "fmt" "net/http" "os/exec" + "strings" "testing" + "time" "github.com/gavv/httpexpect/v2" + . "github.com/onsi/gomega" ) var ( token = "edd1c9f034335f136f87ad84b625c8f1" - host = "http://10.253.0.11:9080" + host = "http://127.0.0.1:9080" ) type httpTestCase struct { @@ -37,6 +41,7 @@ type httpTestCase struct { Body string Headers map[string]string ExpectStatus int + ExpectBody string } func caseCheck(tc httpTestCase) { @@ -51,6 +56,7 @@ func caseCheck(tc httpTestCase) { } if req == nil { + fmt.Printf("%#v", tc) panic("fail to init request") } for key, val := range tc.Headers { @@ -64,16 +70,24 @@ func caseCheck(tc httpTestCase) { if tc.ExpectStatus != 0 { resp.Status(tc.ExpectStatus) } + + if tc.ExpectBody != "" { + resp.Body().Contains(tc.ExpectBody) + } } func setRoute(e *httpexpect.Expect, expectStatus int) { caseCheck(httpTestCase{ E: e, + Method: http.MethodPut, Path: "/apisix/admin/routes/1", Headers: map[string]string{"X-API-KEY": token}, Body: `{ "uri": "/hello", "host": "foo.com", + "plugins": { + "prometheus": {} + }, "upstream": { "nodes": { "bar.org": 1 @@ -88,28 +102,72 @@ func setRoute(e *httpexpect.Expect, expectStatus int) { func getRoute(e *httpexpect.Expect, expectStatus int) { caseCheck(httpTestCase{ E: e, + Method: http.MethodGet, Path: "/hello", Headers: map[string]string{"Host": "foo.com"}, ExpectStatus: expectStatus, }) } +func deleteRoute(e *httpexpect.Expect, expectStatus int) { + caseCheck(httpTestCase{ + E: e, + Method: http.MethodDelete, + Path: "/apisix/admin/routes/1", + Headers: map[string]string{"X-API-KEY": token}, + ExpectStatus: expectStatus, + }) +} + +func getPrometheusMetric(e *httpexpect.Expect, expectEtcd int) { + caseCheck(httpTestCase{ + E: e, + Method: http.MethodGet, + Path: "/apisix/prometheus/metrics", + ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd), + }) +} + +func runCommand(t *testing.T, cmd string) string { + out, err := exec.Command("bash", "-c", cmd).CombinedOutput() + if err != nil { + t.Fatalf("fail to run command %s: %s, %s", cmd, err.Error(), out) + } + return string(out) +} + func TestGetSuccessWhenEtcdKilled(t *testing.T) { + g := NewWithT(t) e := httpexpect.New(t, host) // check if everything works - setRoute(e, http.StatusOK) + setRoute(e, http.StatusCreated) getRoute(e, http.StatusOK) + getPrometheusMetric(e, 1) + + podName := runCommand(t, "kubectl get pod -l app=apisix-gw -o 'jsonpath={..metadata.name}'") + t.Run("error log not contains etcd error", func(t *testing.T) { + errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName)) + g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeFalse()) + }) // TODO: use client-go // apply chaos to kill all etcd pods - _, err := exec.Command("kubectl apply kill-etcd.yaml").CombinedOutput() - if err != nil { - panic("fail to apply chaos yaml") - } + t.Run("kill all etcd pods", func(t *testing.T) { + _ = runCommand(t, "kubectl apply -f kill-etcd.yaml") + time.Sleep(3 * time.Second) + }) // fail to set route since etcd is all killed // while get route could still succeed setRoute(e, http.StatusInternalServerError) getRoute(e, http.StatusOK) + getPrometheusMetric(e, 0) + + t.Run("error log contains etcd error", func(t *testing.T) { + errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName)) + g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeTrue()) + }) + + //deleteRoute(e, http.StatusOK) } diff --git a/t/chaos/utils.sh b/t/chaos/utils.sh old mode 100644 new mode 100755 index 1de24e05ddd8..7a250b813423 --- a/t/chaos/utils.sh +++ b/t/chaos/utils.sh @@ -19,30 +19,41 @@ set -ex -startMinikube() { +start_minikube() { curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb sudo dpkg -i --force-architecture minikube_latest_amd64.deb - sudo mv ./minikube /usr/local/bin/minikube minikube start } +modify_config() { + DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}') + echo "dns_resolver: + - ${DNS_IP} +etcd: + host: + - \"http://etcd-cluster-client.default.svc.cluster.local:2379\" " > ./conf/config.yaml +} + ensure_pods_ready() { local app=$1 local status=$2 + local retries=$3 count=0 - while [ -n "$(kubectl get pods -l app=${app} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != ${status})" ]; + while [[ $(kubectl get pods -l app=${app} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != ${status} ]]; do echo "Waiting for pod running" && sleep 10; ((count=count+1)) - # waiting for 300s, or timeout - if [ $count -gt 30 ]; then + if [ $count -gt ${retries} ]; then printf "Waiting for pod status running timeout\n" + kubectl describe pod -l app=${app} + printf "\n\n" + kubectl logs -l app=${app} exit 1 fi done