From 086cc82723958cdde385974f5e3750faf588f7e6 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Tue, 24 Oct 2017 18:22:35 +0200 Subject: [PATCH 1/5] Fixup variable names in DC controller --- .../deploymentconfig_controller.go | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go b/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go index fc4b93f4aefc..a337c1e234e0 100644 --- a/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go +++ b/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go @@ -121,16 +121,16 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) return c.updateStatus(config, existingDeployments) } - latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) + latestExists, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) - if !latestIsDeployed { + if !latestExists { if err := c.cancelRunningRollouts(config, existingDeployments, cm); err != nil { return err } } configCopy := config.DeepCopy() // Process triggers and start an initial rollouts - shouldTrigger, shouldSkip := triggerActivated(configCopy, latestIsDeployed, latestDeployment, c.codec) + shouldTrigger, shouldSkip := triggerActivated(configCopy, latestExists, latestDeployment, c.codec) if !shouldSkip && shouldTrigger { configCopy.Status.LatestVersion++ return c.updateStatus(configCopy, existingDeployments) @@ -141,18 +141,7 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) } // If the latest deployment already exists, reconcile existing deployments // and return early. - if latestIsDeployed { - // If the latest deployment is still running, try again later. We don't - // want to compete with the deployer. - if !deployutil.IsTerminatedDeployment(latestDeployment) { - return c.updateStatus(config, existingDeployments) - } - - return c.reconcileDeployments(existingDeployments, config, cm) - } - // If the latest deployment already exists, reconcile existing deployments - // and return early. - if latestIsDeployed { + if latestExists { // If the latest deployment is still running, try again later. We don't // want to compete with the deployer. if !deployutil.IsTerminatedDeployment(latestDeployment) { @@ -530,7 +519,7 @@ func (c *DeploymentConfigController) cleanupOldDeployments(existingDeployments [ // triggers were activated (config change or image change). The first bool indicates that // the triggers are active and second indicates if we should skip the rollout because we // are waiting for the trigger to complete update (waiting for image for example). -func triggerActivated(config *deployapi.DeploymentConfig, latestIsDeployed bool, latestDeployment *v1.ReplicationController, codec runtime.Codec) (bool, bool) { +func triggerActivated(config *deployapi.DeploymentConfig, latestExists bool, latestDeployment *v1.ReplicationController, codec runtime.Codec) (bool, bool) { if config.Spec.Paused { return false, false } @@ -569,7 +558,7 @@ func triggerActivated(config *deployapi.DeploymentConfig, latestIsDeployed bool, } // Wait for the RC to be created - if !latestIsDeployed { + if !latestExists { return false, true } From 705e69b39be321542d0d6d3e304c6cf653805555 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Tue, 24 Oct 2017 18:24:08 +0200 Subject: [PATCH 2/5] Make deployment test reproducible when randomness is involved --- test/extended/deployments/deployments.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/extended/deployments/deployments.go b/test/extended/deployments/deployments.go index c069b181e721..a39fce9dc98c 100644 --- a/test/extended/deployments/deployments.go +++ b/test/extended/deployments/deployments.go @@ -59,12 +59,13 @@ var _ = g.Describe("[Feature:DeploymentConfig] deploymentconfigs", func() { _, err := oc.Run("create").Args("-f", simpleDeploymentFixture).Output() o.Expect(err).NotTo(o.HaveOccurred()) + r := rand.New(rand.NewSource(g.GinkgoRandomSeed())) iterations := 15 for i := 0; i < iterations; i++ { - if rand.Float32() < 0.2 { - time.Sleep(time.Duration(rand.Float32() * rand.Float32() * float32(time.Second))) + if r.Float32() < 0.2 { + time.Sleep(time.Duration(r.Float32() * r.Float32() * float32(time.Second))) } - switch n := rand.Float32(); { + switch n := r.Float32(); { case n < 0.4: // trigger a new deployment @@ -111,7 +112,7 @@ var _ = g.Describe("[Feature:DeploymentConfig] deploymentconfigs", func() { for _, pod := range pods { e2e.Logf("%02d: deleting deployer pod %s", i, pod.Name) options := metav1.NewDeleteOptions(0) - if rand.Float32() < 0.5 { + if r.Float32() < 0.5 { options = nil } if err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Delete(pod.Name, options); err != nil { From 020235fbabca30fa81d0f8e1771615fdf1c725d6 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Tue, 24 Oct 2017 19:29:28 +0200 Subject: [PATCH 3/5] Add asynchronous deployer pod invariant checker for every test --- .../deployer/deployer_controller.go | 3 +- test/extended/deployments/deployments.go | 48 ++++- test/extended/deployments/util.go | 165 +++++++++++++++++- 3 files changed, 207 insertions(+), 9 deletions(-) diff --git a/pkg/apps/controller/deployer/deployer_controller.go b/pkg/apps/controller/deployer/deployer_controller.go index e944b2f76703..d0c22a201547 100755 --- a/pkg/apps/controller/deployer/deployer_controller.go +++ b/pkg/apps/controller/deployer/deployer_controller.go @@ -379,7 +379,8 @@ func (c *DeploymentController) makeDeployerPod(deployment *v1.ReplicationControl ObjectMeta: metav1.ObjectMeta{ Name: deployutil.DeployerPodNameForDeployment(deployment.Name), Annotations: map[string]string{ - deployapi.DeploymentAnnotation: deployment.Name, + deployapi.DeploymentAnnotation: deployment.Name, + deployapi.DeploymentConfigAnnotation: deployutil.DeploymentConfigNameFor(deployment), }, Labels: map[string]string{ deployapi.DeployerPodForDeploymentLabel: deployment.Name, diff --git a/test/extended/deployments/deployments.go b/test/extended/deployments/deployments.go index a39fce9dc98c..a3046584352c 100644 --- a/test/extended/deployments/deployments.go +++ b/test/extended/deployments/deployments.go @@ -1,6 +1,7 @@ package deployments import ( + "context" "errors" "fmt" "math/rand" @@ -27,10 +28,55 @@ import ( const deploymentRunTimeout = 5 * time.Minute const deploymentChangeTimeout = 30 * time.Second +type dicEntry struct { + dic *deployerPodInvariantChecker + ctx context.Context + cancel func() +} + var _ = g.Describe("[Feature:DeploymentConfig] deploymentconfigs", func() { defer g.GinkgoRecover() + + dicMap := make(map[string]dicEntry) + var oc *exutil.CLI + + g.JustBeforeEach(func() { + namespace := oc.Namespace() + o.Expect(namespace).NotTo(o.BeEmpty()) + o.Expect(dicMap).NotTo(o.HaveKey(namespace)) + + dic := NewDeployerPodInvariantChecker(namespace, oc.AdminKubeClient()) + ctx, cancel := context.WithCancel(context.Background()) + dic.Start(ctx) + + dicMap[namespace] = dicEntry{ + dic: dic, + ctx: ctx, + cancel: cancel, + } + }) + + // This have to be registered before we create kube framework (NewCLI). + // It is probably a bug with Ginkgo because AfterEach description say innermost will be run first + // but it runs outermost first. + g.AfterEach(func() { + namespace := oc.Namespace() + o.Expect(namespace).NotTo(o.BeEmpty(), "There is something wrong with testing framework or the AfterEach functions have been registered in wrong order") + o.Expect(dicMap).To(o.HaveKey(namespace)) + + // Give some time to the checker to catch up + time.Sleep(2 * time.Second) + + entry := dicMap[namespace] + delete(dicMap, namespace) + + entry.cancel() + entry.dic.Wait() + }) + + oc = exutil.NewCLI("cli-deployment", exutil.KubeConfigPath()) + var ( - oc = exutil.NewCLI("cli-deployment", exutil.KubeConfigPath()) deploymentFixture = exutil.FixturePath("testdata", "deployments", "test-deployment-test.yaml") simpleDeploymentFixture = exutil.FixturePath("testdata", "deployments", "deployment-simple.yaml") customDeploymentFixture = exutil.FixturePath("testdata", "deployments", "custom-deployment.yaml") diff --git a/test/extended/deployments/util.go b/test/extended/deployments/util.go index 89bc106a81aa..1f92a25d8acd 100644 --- a/test/extended/deployments/util.go +++ b/test/extended/deployments/util.go @@ -1,21 +1,28 @@ package deployments import ( + "context" "fmt" "io/ioutil" "reflect" "sort" "strings" + "sync" "time" + "github.com/davecgh/go-spew/spew" "github.com/ghodss/yaml" + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" kapi "k8s.io/kubernetes/pkg/api" kapiv1 "k8s.io/kubernetes/pkg/api/v1" @@ -426,6 +433,24 @@ func rCConditionFromMeta(condition func(metav1.Object) (bool, error)) func(rc *c } } +func waitForPodModification(oc *exutil.CLI, namespace string, name string, timeout time.Duration, resourceVersion string, condition func(pod *corev1.Pod) (bool, error)) (*corev1.Pod, error) { + watcher, err := oc.KubeClient().CoreV1().Pods(namespace).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: name, ResourceVersion: resourceVersion})) + if err != nil { + return nil, err + } + + event, err := watch.Until(timeout, watcher, func(event watch.Event) (bool, error) { + if event.Type != watch.Modified && (resourceVersion == "" && event.Type != watch.Added) { + return true, fmt.Errorf("different kind of event appeared while waiting for Pod modification: event: %#v", event) + } + return condition(event.Object.(*corev1.Pod)) + }) + if err != nil { + return nil, err + } + return event.Object.(*corev1.Pod), nil +} + func waitForRCModification(oc *exutil.CLI, namespace string, name string, timeout time.Duration, resourceVersion string, condition func(rc *corev1.ReplicationController) (bool, error)) (*corev1.ReplicationController, error) { watcher, err := oc.KubeClient().CoreV1().ReplicationControllers(namespace).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: name, ResourceVersion: resourceVersion})) if err != nil { @@ -433,8 +458,8 @@ func waitForRCModification(oc *exutil.CLI, namespace string, name string, timeou } event, err := watch.Until(timeout, watcher, func(event watch.Event) (bool, error) { - if event.Type != watch.Modified { - return false, fmt.Errorf("different kind of event appeared while waiting for modification: event: %#v", event) + if event.Type != watch.Modified && (resourceVersion == "" && event.Type != watch.Added) { + return true, fmt.Errorf("different kind of event appeared while waiting for RC modification: event: %#v", event) } return condition(event.Object.(*corev1.ReplicationController)) }) @@ -454,17 +479,14 @@ func waitForDCModification(oc *exutil.CLI, namespace string, name string, timeou } event, err := watch.Until(timeout, watcher, func(event watch.Event) (bool, error) { - if event.Type != watch.Modified { - return false, fmt.Errorf("different kind of event appeared while waiting for modification: event: %#v", event) + if event.Type != watch.Modified && (resourceVersion == "" && event.Type != watch.Added) { + return true, fmt.Errorf("different kind of event appeared while waiting for DC modification: event: %#v", event) } return condition(event.Object.(*deployapi.DeploymentConfig)) }) if err != nil { return nil, err } - if event.Type != watch.Modified { - return nil, fmt.Errorf("waiting for DC modification failed: event: %v", event) - } return event.Object.(*deployapi.DeploymentConfig), nil } @@ -623,3 +645,132 @@ func readDCFixtureOrDie(path string) *deployapi.DeploymentConfig { } return data } + +type deployerPodInvariantChecker struct { + ctx context.Context + wg sync.WaitGroup + namespace string + client kubernetes.Interface + cache map[string][]*corev1.Pod +} + +func NewDeployerPodInvariantChecker(namespace string, client kubernetes.Interface) *deployerPodInvariantChecker { + return &deployerPodInvariantChecker{ + namespace: namespace, + client: client, + cache: make(map[string][]*corev1.Pod), + } +} + +func (d *deployerPodInvariantChecker) getCacheKey(pod *corev1.Pod) string { + dcName, found := pod.Annotations[deployapi.DeploymentConfigAnnotation] + o.Expect(found).To(o.BeTrue(), fmt.Sprintf("internal error - deployment is missing %q annotation\npod: %#v", deployapi.DeploymentConfigAnnotation, pod)) + o.Expect(dcName).NotTo(o.BeEmpty()) + + return fmt.Sprintf("%s/%s", pod.Namespace, dcName) +} +func (d *deployerPodInvariantChecker) getPodIndex(list []*corev1.Pod, pod *corev1.Pod) int { + for i, p := range list { + if p.Name == pod.Name && p.Namespace == pod.Namespace { + // Internal check + o.Expect(p.UID).To(o.Equal(pod.UID)) + return i + } + } + + // Internal check + o.Expect(fmt.Errorf("couldn't find pod %#v \n\n in list %#v", pod, list)).NotTo(o.HaveOccurred()) + return -1 +} + +func (d *deployerPodInvariantChecker) checkInvariants(dc string, pods []*corev1.Pod) { + var unterminatedPods []*corev1.Pod + for _, pod := range pods { + if pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { + unterminatedPods = append(unterminatedPods, pod) + } + } + + // INVARIANT: There can be no more than one unterminated deployer pod present + message := fmt.Sprintf("Deployer pod invariant broken! More than one unterminated deployer pod exists for DC %s!", dc) + o.Expect(len(unterminatedPods)).To(o.BeNumerically("<=", 1), spew.Sprintf(`%v: %s + List of unterminated pods: %#+v + `, time.Now(), message, unterminatedPods)) +} + +func (d *deployerPodInvariantChecker) AddPod(pod *corev1.Pod) { + key := d.getCacheKey(pod) + d.cache[key] = append(d.cache[key], pod) + + d.checkInvariants(key, d.cache[key]) +} + +func (d *deployerPodInvariantChecker) RemovePod(pod *corev1.Pod) { + key := d.getCacheKey(pod) + index := d.getPodIndex(d.cache[key], pod) + + d.cache[key] = append(d.cache[key][:index], d.cache[key][index+1:]...) + + d.checkInvariants(key, d.cache[key]) +} + +func (d *deployerPodInvariantChecker) UpdatePod(pod *corev1.Pod) { + key := d.getCacheKey(pod) + index := d.getPodIndex(d.cache[key], pod) + + // Check for sanity. + // This is not paranoid; kubelet has already been broken this way: + // https://github.com/openshift/origin/issues/17011 + oldPhase := d.cache[key][index].Status.Phase + oldPhaseIsTerminated := oldPhase == corev1.PodSucceeded || oldPhase == corev1.PodFailed + o.Expect(oldPhaseIsTerminated && pod.Status.Phase != oldPhase).To(o.BeFalse(), + fmt.Sprintf("%v: detected deployer pod transition from terminated phase: %q -> %q", time.Now(), oldPhase, pod.Status.Phase)) + + d.cache[key][index] = pod + + d.checkInvariants(key, d.cache[key]) +} + +func (d *deployerPodInvariantChecker) doChecking() { + defer g.GinkgoRecover() + + watcher, err := d.client.CoreV1().Pods(d.namespace).Watch(metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + defer d.wg.Done() + defer watcher.Stop() + + for { + select { + case <-d.ctx.Done(): + return + case event := <-watcher.ResultChan(): + t := event.Type + if t != watch.Added && t != watch.Modified && t != watch.Deleted { + o.Expect(fmt.Errorf("unexpected event: %#v", event)).NotTo(o.HaveOccurred()) + } + pod := event.Object.(*corev1.Pod) + if !strings.HasSuffix(pod.Name, "-deploy") { + continue + } + + switch t { + case watch.Added: + d.AddPod(pod) + case watch.Modified: + d.UpdatePod(pod) + case watch.Deleted: + d.RemovePod(pod) + } + } + } +} + +func (d *deployerPodInvariantChecker) Start(ctx context.Context) { + d.ctx = ctx + go d.doChecking() + d.wg.Add(1) +} + +func (d *deployerPodInvariantChecker) Wait() { + d.wg.Wait() +} From 62b1cbc65aacb0d391052665ab0917de223312d0 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Tue, 24 Oct 2017 18:28:55 +0200 Subject: [PATCH 4/5] Enable back the old check for multiple deployer pods temporarily disabled in https://github.com/openshift/origin/pull/16956 --- test/extended/deployments/util.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/extended/deployments/util.go b/test/extended/deployments/util.go index 1f92a25d8acd..06e36f73c489 100644 --- a/test/extended/deployments/util.go +++ b/test/extended/deployments/util.go @@ -162,10 +162,9 @@ func checkDeploymentInvariants(dc *deployapi.DeploymentConfig, rcs []*corev1.Rep running.Insert(k) } } - // FIXME: enable this check when we fix the controllers - //if running.Len() > 1 { - // return fmt.Errorf("found multiple running deployments: %v", running.List()) - //} + if running.Len() > 1 { + return fmt.Errorf("found multiple running deployments: %v", running.List()) + } sawStatus := sets.NewString() statuses := []string{} for _, rc := range rcs { From ba495e5dcb84ddbc4d71fad2ceba7a22eab24c88 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Mon, 30 Oct 2017 17:15:45 +0100 Subject: [PATCH 5/5] Add some test to stress test deployer pod invariants --- test/extended/deployments/deployments.go | 217 +++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/test/extended/deployments/deployments.go b/test/extended/deployments/deployments.go index a3046584352c..cb2cb8afc15c 100644 --- a/test/extended/deployments/deployments.go +++ b/test/extended/deployments/deployments.go @@ -1160,4 +1160,221 @@ var _ = g.Describe("[Feature:DeploymentConfig] deploymentconfigs", func() { }) }) }) + + g.Describe("keep the deployer pod invariant valid [Conformance]", func() { + dcName := "deployment-simple" + + g.AfterEach(func() { + failureTrap(oc, dcName, g.CurrentGinkgoTestDescription().Failed) + }) + + g.It("should deal with cancellation of running deployment", func() { + namespace := oc.Namespace() + + g.By("creating DC") + dc, err := readDCFixture(simpleDeploymentFixture) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(dc.Name).To(o.Equal(dcName)) + + dc.Spec.Replicas = 1 + // Make sure the deployer pod doesn't end too soon + dc.Spec.MinReadySeconds = 60 + dc, err = oc.AppsClient().Apps().DeploymentConfigs(namespace).Create(dc) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("waiting for RC to be created") + dc, err = waitForDCModification(oc, namespace, dcName, deploymentRunTimeout, + dc.GetResourceVersion(), func(config *deployapi.DeploymentConfig) (bool, error) { + cond := deployutil.GetDeploymentCondition(config.Status, deployapi.DeploymentProgressing) + if cond != nil && cond.Reason == deployapi.NewReplicationControllerReason { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(dc.Status.LatestVersion).To(o.BeEquivalentTo(1)) + + g.By("waiting for deployer pod to be running") + rc, err := waitForRCModification(oc, namespace, deployutil.LatestDeploymentNameForConfig(dc), deploymentRunTimeout, + "", func(currentRC *kapiv1.ReplicationController) (bool, error) { + if deployutil.DeploymentStatusFor(currentRC) == deployapi.DeploymentStatusRunning { + return true, nil + } + return false, nil + }) + + g.By("canceling the deployment") + rc, err = oc.KubeClient().CoreV1().ReplicationControllers(namespace).Patch( + deployutil.LatestDeploymentNameForConfig(dc), types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"annotations":{%q: %q, %q: %q}}}`, + deployapi.DeploymentCancelledAnnotation, deployapi.DeploymentCancelledAnnotationValue, + deployapi.DeploymentStatusReasonAnnotation, deployapi.DeploymentCancelledByUser, + ))) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(deployutil.DeploymentVersionFor(rc)).To(o.Equal(dc.Status.LatestVersion)) + + g.By("redeploying immediately by config change") + o.Expect(dc.Spec.Template.Annotations["foo"]).NotTo(o.Equal("bar")) + dc, err = oc.AppsClient().Apps().DeploymentConfigs(dc.Namespace).Patch(dc.Name, types.StrategicMergePatchType, + []byte(`{"spec":{"template":{"metadata":{"annotations":{"foo": "bar"}}}}}`)) + o.Expect(err).NotTo(o.HaveOccurred()) + dc, err = waitForDCModification(oc, namespace, dcName, deploymentRunTimeout, + dc.GetResourceVersion(), func(config *deployapi.DeploymentConfig) (bool, error) { + if config.Status.LatestVersion == 2 { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for deployment pod to be running + rc, err = waitForRCModification(oc, namespace, deployutil.LatestDeploymentNameForConfig(dc), deploymentRunTimeout, + "", func(currentRC *kapiv1.ReplicationController) (bool, error) { + if deployutil.DeploymentStatusFor(currentRC) == deployapi.DeploymentStatusRunning { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("should deal with config change in case the deployment is still running", func() { + namespace := oc.Namespace() + + g.By("creating DC") + dc, err := readDCFixture(simpleDeploymentFixture) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(dc.Name).To(o.Equal(dcName)) + + dc.Spec.Replicas = 1 + // Make sure the deployer pod doesn't end too soon + dc.Spec.MinReadySeconds = 60 + dc, err = oc.AppsClient().Apps().DeploymentConfigs(namespace).Create(dc) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("waiting for RC to be created") + dc, err = waitForDCModification(oc, namespace, dc.Name, deploymentRunTimeout, + dc.GetResourceVersion(), func(config *deployapi.DeploymentConfig) (bool, error) { + cond := deployutil.GetDeploymentCondition(config.Status, deployapi.DeploymentProgressing) + if cond != nil && cond.Reason == deployapi.NewReplicationControllerReason { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(dc.Status.LatestVersion).To(o.BeEquivalentTo(1)) + + g.By("waiting for deployer pod to be running") + _, err = waitForRCModification(oc, namespace, deployutil.LatestDeploymentNameForConfig(dc), deploymentRunTimeout, + "", func(currentRC *kapiv1.ReplicationController) (bool, error) { + if deployutil.DeploymentStatusFor(currentRC) == deployapi.DeploymentStatusRunning { + return true, nil + } + return false, nil + }) + + g.By("redeploying immediately by config change") + o.Expect(dc.Spec.Template.Annotations["foo"]).NotTo(o.Equal("bar")) + dc, err = oc.AppsClient().Apps().DeploymentConfigs(dc.Namespace).Patch(dc.Name, types.StrategicMergePatchType, + []byte(`{"spec":{"template":{"metadata":{"annotations":{"foo": "bar"}}}}}`)) + o.Expect(err).NotTo(o.HaveOccurred()) + dc, err = waitForDCModification(oc, namespace, dcName, deploymentRunTimeout, + dc.GetResourceVersion(), func(config *deployapi.DeploymentConfig) (bool, error) { + if config.Status.LatestVersion == 2 { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for deployment pod to be running + _, err = waitForRCModification(oc, namespace, deployutil.LatestDeploymentNameForConfig(dc), deploymentRunTimeout, + "", func(currentRC *kapiv1.ReplicationController) (bool, error) { + if deployutil.DeploymentStatusFor(currentRC) == deployapi.DeploymentStatusRunning { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("should deal with cancellation after deployer pod succeeded", func() { + namespace := oc.Namespace() + + g.By("creating DC") + dc, err := readDCFixture(simpleDeploymentFixture) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(dc.Name).To(o.Equal(dcName)) + + dc.Spec.Replicas = 1 + // Make sure the deployer pod doesn't immediately + dc.Spec.MinReadySeconds = 3 + dc, err = oc.AppsClient().Apps().DeploymentConfigs(namespace).Create(dc) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("waiting for RC to be created") + dc, err = waitForDCModification(oc, namespace, dc.Name, deploymentRunTimeout, + dc.GetResourceVersion(), func(config *deployapi.DeploymentConfig) (bool, error) { + cond := deployutil.GetDeploymentCondition(config.Status, deployapi.DeploymentProgressing) + if cond != nil && cond.Reason == deployapi.NewReplicationControllerReason { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(dc.Status.LatestVersion).To(o.BeEquivalentTo(1)) + + rcName := deployutil.LatestDeploymentNameForConfig(dc) + + g.By("waiting for deployer to be completed") + _, err = waitForPodModification(oc, namespace, + deployutil.DeployerPodNameForDeployment(rcName), + deploymentRunTimeout, "", + func(pod *kapiv1.Pod) (bool, error) { + switch pod.Status.Phase { + case kapiv1.PodSucceeded: + return true, nil + case kapiv1.PodFailed: + return true, errors.New("pod failed") + default: + return false, nil + } + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("canceling the deployment") + rc, err := oc.KubeClient().CoreV1().ReplicationControllers(namespace).Patch( + rcName, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"annotations":{%q: %q, %q: %q}}}`, + deployapi.DeploymentCancelledAnnotation, deployapi.DeploymentCancelledAnnotationValue, + deployapi.DeploymentStatusReasonAnnotation, deployapi.DeploymentCancelledByUser, + ))) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(deployutil.DeploymentVersionFor(rc)).To(o.BeEquivalentTo(1)) + + g.By("redeploying immediately by config change") + o.Expect(dc.Spec.Template.Annotations["foo"]).NotTo(o.Equal("bar")) + dc, err = oc.AppsClient().Apps().DeploymentConfigs(dc.Namespace).Patch(dc.Name, types.StrategicMergePatchType, + []byte(`{"spec":{"template":{"metadata":{"annotations":{"foo": "bar"}}}}}`)) + o.Expect(err).NotTo(o.HaveOccurred()) + dc, err = waitForDCModification(oc, namespace, dcName, deploymentRunTimeout, + dc.GetResourceVersion(), func(config *deployapi.DeploymentConfig) (bool, error) { + if config.Status.LatestVersion == 2 { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for deployment pod to be running + _, err = waitForRCModification(oc, namespace, deployutil.LatestDeploymentNameForConfig(dc), deploymentRunTimeout, + "", func(currentRC *kapiv1.ReplicationController) (bool, error) { + if deployutil.DeploymentStatusFor(currentRC) == deployapi.DeploymentStatusRunning { + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + }) })