Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

e2e case for container kill and container kill pause #437

Merged
merged 4 commits into from
Apr 21, 2020
Merged

e2e case for container kill and container kill pause #437

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a776ee7
add container kill e2e
yeya24 Apr 20, 2020
cbe1539
add container kill e2e and pause
yeya24 Apr 20, 2020
d6810bb
cleanup
yeya24 Apr 21, 2020
42224b2
Merge branch 'master' into e2e-case
cwen0 Apr 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
Expand Down Expand Up @@ -499,8 +497,6 @@ github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc8
github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
github.com/mgechev/revive v1.0.2-0.20200225072153-6219ca02fffb h1:EabZ4SffLYB6FcYN8VDMk1TCMahjhEhEqKcOxBNbPmY=
github.com/mgechev/revive v1.0.2-0.20200225072153-6219ca02fffb/go.mod h1:E9j8UNyHeYo/uUXIIUOAehxf5B69UwZ5u3qj7wEn8J0=
github.com/mgechev/revive v1.0.2 h1:v0NxxQ7fSFz/u1NQydPo6EGdq7va0J1BtsZmae6kzUg=
github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo=
github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY=
github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
Expand Down Expand Up @@ -833,8 +829,6 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ym
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200320220750-118fecf932d8 h1:1+zQlQqEEhUeStBTi653GZAnAuivZq/2hz+Iz+OP7rg=
golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -884,8 +878,6 @@ golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down Expand Up @@ -939,7 +931,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200309202150-20ab64c0d93f h1:NbrfHxef+IfdI86qCgO/1Siq1BuMH2xG0NqgvCguRhQ=
golang.org/x/tools v0.0.0-20200309202150-20ab64c0d93f/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand All @@ -963,8 +954,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20200306153348-d950eab6f860 h1:QmnwU8dKvY8c/vZikd2jhBNwrrGS5qeyK/2Aeeh9Grk=
google.golang.org/genproto v0.0.0-20200306153348-d950eab6f860/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200323114720-3f67cca34472 h1:XRuIAeTRoXziYGYTVer+YGxVXQBiOhZ8+SpNELP73oQ=
google.golang.org/genproto v0.0.0-20200323114720-3f67cca34472/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
Expand Down
219 changes: 214 additions & 5 deletions test/e2e/chaos/basic.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,12 @@ import (
restClient "k8s.io/client-go/rest"
"k8s.io/klog"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/pingcap/chaos-mesh/api/v1alpha1"
chaosmeshv1alpha1 "github.com/pingcap/chaos-mesh/api/v1alpha1"
"github.com/pingcap/chaos-mesh/test/pkg/fixture"

// load pprof
_ "net/http/pprof"
)

const (
Expand Down Expand Up @@ -204,7 +202,6 @@ var _ = ginkgo.Describe("[chaos-mesh] Basic", func() {
pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")

dur := "5s"
podKillChaos := &v1alpha1.PodChaos{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx-kill",
Expand All @@ -217,7 +214,7 @@ var _ = ginkgo.Describe("[chaos-mesh] Basic", func() {
},
Action: v1alpha1.PodKillAction,
Mode: v1alpha1.OnePodMode,
Duration: &dur,
Duration: pointer.StringPtr("5s"),
Scheduler: &v1alpha1.SchedulerSpec{
Cron: "@every 10s",
},
Expand Down Expand Up @@ -251,6 +248,7 @@ var _ = ginkgo.Describe("[chaos-mesh] Basic", func() {
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
chaos := &v1alpha1.PodChaos{}
err = cli.Get(ctx, chaosKey, chaos)
framework.ExpectNoError(err, "get nginx pods error")
if chaos.Status.Experiment.Phase == chaosmeshv1alpha1.ExperimentPhasePaused {
return true, nil
}
Expand Down Expand Up @@ -280,6 +278,7 @@ var _ = ginkgo.Describe("[chaos-mesh] Basic", func() {
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
chaos := &v1alpha1.PodChaos{}
err = cli.Get(ctx, chaosKey, chaos)
framework.ExpectNoError(err, "get nginx pods error")
if chaos.Status.Experiment.Phase == chaosmeshv1alpha1.ExperimentPhaseRunning {
return true, nil
}
Expand All @@ -297,6 +296,216 @@ var _ = ginkgo.Describe("[chaos-mesh] Basic", func() {

cancel()
})

ginkgo.It("ContainerKill", func() {
ctx, cancel := context.WithCancel(context.Background())
nd := fixture.NewCommonNginxDeployment("nginx", ns, 1)
_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
framework.ExpectNoError(err, "create nginx deployment error")
err = waitDeploymentReady("nginx", ns, kubeCli)
framework.ExpectNoError(err, "wait nginx deployment ready error")

containerKillChaos := &v1alpha1.PodChaos{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx-container-kill",
Namespace: ns,
},
Spec: v1alpha1.PodChaosSpec{
Selector: v1alpha1.SelectorSpec{
Namespaces: []string{
ns,
},
LabelSelectors: map[string]string{
"app": "nginx",
},
},
Action: v1alpha1.ContainerKillAction,
Mode: v1alpha1.OnePodMode,
ContainerName: "nginx",
Scheduler: &v1alpha1.SchedulerSpec{
Cron: "@every 10s",
},
},
}
err = cli.Create(ctx, containerKillChaos)
framework.ExpectNoError(err, "create container kill chaos error")

err = wait.PollImmediate(3*time.Second, 5*time.Minute, func() (done bool, err error) {
listOption := metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(map[string]string{
"app": "nginx",
}).String(),
}
pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
if err != nil {
return false, nil
}
if len(pods.Items) != 1 {
return false, nil
}
pod := pods.Items[0]
for _, cs := range pod.Status.ContainerStatuses {
if cs.Name == "nginx" && cs.Ready == false && cs.LastTerminationState.Terminated != nil {
return true, nil
}
}
return false, nil
})

err = cli.Delete(ctx, containerKillChaos)
framework.ExpectNoError(err, "failed to delete container kill chaos")

klog.Infof("success to perform container kill")
err = wait.PollImmediate(3*time.Second, 5*time.Minute, func() (done bool, err error) {
listOption := metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(map[string]string{
"app": "nginx",
}).String(),
}
pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
if err != nil {
return false, nil
}
if len(pods.Items) != 1 {
return false, nil
}
pod := pods.Items[0]
for _, cs := range pod.Status.ContainerStatuses {
if cs.Name == "nginx" && cs.Ready == true && cs.State.Running != nil {
return true, nil
}
}
return false, nil
})
framework.ExpectNoError(err, "container kill recover failed")

cancel()
})

ginkgo.It("PauseContainerKill", func() {
ctx, cancel := context.WithCancel(context.Background())
nd := fixture.NewCommonNginxDeployment("nginx", ns, 3)
_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
framework.ExpectNoError(err, "create nginx deployment error")
err = waitDeploymentReady("nginx", ns, kubeCli)
framework.ExpectNoError(err, "wait nginx deployment ready error")

var pods *corev1.PodList
var newPods *corev1.PodList
listOption := metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(map[string]string{
"app": "nginx",
}).String(),
}
pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")

// Get the running nginx container ID
containerID := pods.Items[0].Status.ContainerStatuses[0].ContainerID

containerKillChaos := &v1alpha1.PodChaos{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx-container-kill",
Namespace: ns,
},
Spec: v1alpha1.PodChaosSpec{
Selector: v1alpha1.SelectorSpec{
Namespaces: []string{
ns,
},
LabelSelectors: map[string]string{
"app": "nginx",
},
},
Action: v1alpha1.ContainerKillAction,
Mode: v1alpha1.OnePodMode,
ContainerName: "nginx",
Duration: pointer.StringPtr("5s"),
Scheduler: &v1alpha1.SchedulerSpec{
Cron: "@every 10s",
},
},
}
err = cli.Create(ctx, containerKillChaos)
framework.ExpectNoError(err, "create container kill chaos error")

chaosKey := types.NamespacedName{
Namespace: ns,
Name: "nginx-container-kill",
}

// nginx container is killed as expected
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
newPods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")
return containerID != newPods.Items[0].Status.ContainerStatuses[0].ContainerID, nil
})
framework.ExpectNoError(err, "wait container kill failed")

// pause experiment
var mergePatch []byte
mergePatch, _ = json.Marshal(map[string]interface{}{
"spec": map[string]interface{}{
"paused": true,
},
})
err = cli.Patch(ctx, containerKillChaos, client.ConstantPatch(types.MergePatchType, mergePatch))
framework.ExpectNoError(err, "patch pause pod chaos error")
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
chaos := &v1alpha1.PodChaos{}
err = cli.Get(ctx, chaosKey, chaos)
framework.ExpectNoError(err, "get pod chaos error")
if chaos.Status.Experiment.Phase == chaosmeshv1alpha1.ExperimentPhasePaused {
return true, nil
}
return false, err
})
framework.ExpectNoError(err, "check paused chaos failed")

// wait for 1 minutes and check whether nginx container will be killed or not
pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")
containerID = pods.Items[0].Status.ContainerStatuses[0].ContainerID
err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
newPods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")
return containerID != newPods.Items[0].Status.ContainerStatuses[0].ContainerID, nil
})
framework.ExpectError(err, "wait container not killed failed")
framework.ExpectEqual(err.Error(), wait.ErrWaitTimeout.Error())

// resume experiment
mergePatch, _ = json.Marshal(map[string]interface{}{
"spec": map[string]interface{}{
"paused": false,
},
})
err = cli.Patch(ctx, containerKillChaos, client.ConstantPatch(types.MergePatchType, mergePatch))
framework.ExpectNoError(err, "patch resume pod chaos error")
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
chaos := &v1alpha1.PodChaos{}
err = cli.Get(ctx, chaosKey, chaos)
framework.ExpectNoError(err, "get pod chaos error")
if chaos.Status.Experiment.Phase == chaosmeshv1alpha1.ExperimentPhaseRunning {
return true, nil
}
return false, err
})
framework.ExpectNoError(err, "check resumed chaos failed")

// nginx container is killed by resumed experiment
pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")
containerID = pods.Items[0].Status.ContainerStatuses[0].ContainerID
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
newPods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
framework.ExpectNoError(err, "get nginx pods error")
return containerID != newPods.Items[0].Status.ContainerStatuses[0].ContainerID, nil
})
framework.ExpectNoError(err, "wait container killed failed")

cancel()
})
})

func waitPodRunning(name, namespace string, cli kubernetes.Interface) error {
Expand Down