From 6fb2548c5e68079b4619e983e40459980baa8b90 Mon Sep 17 00:00:00 2001 From: Weiyu Yen Date: Wed, 29 May 2024 20:39:56 -0700 Subject: [PATCH] add retry for update operation Signed-off-by: Weiyu Yen --- .../pytorch/pytorchjob_controller_test.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/controller.v1/pytorch/pytorchjob_controller_test.go b/pkg/controller.v1/pytorch/pytorchjob_controller_test.go index b168a1bf88..b035251fa3 100644 --- a/pkg/controller.v1/pytorch/pytorchjob_controller_test.go +++ b/pkg/controller.v1/pytorch/pytorchjob_controller_test.go @@ -203,14 +203,19 @@ var _ = Describe("PyTorchJob controller", func() { By("Attempting to update the PyTorchJob with a different queue value") updatedJob := &kubeflowv1.PyTorchJob{} - Expect(testK8sClient.Get(ctx, client.ObjectKeyFromObject(job), updatedJob)).Should(Succeed(), "Failed to get PyTorchJob") - - updatedJob.Spec.RunPolicy.SchedulingPolicy.Queue = "test" - err := testK8sClient.Update(ctx, updatedJob) + Eventually(func() bool { + err := testK8sClient.Get(ctx, jobKey, updatedJob) + return err == nil + }, testutil.Timeout, testutil.Interval).Should(BeTrue(), "Failed to get PyTorchJob") - By("Checking that the queue update fails") - Expect(err).To(HaveOccurred(), "Expected an error when updating the queue, but update succeeded") - Expect(err.Error()).To(ContainSubstring("spec.runPolicy.schedulingPolicy.queue is immutable"), "The error message did not contain the expected message") + Eventually(func() bool { + updatedJob.Spec.RunPolicy.SchedulingPolicy.Queue = "test" + err := testK8sClient.Update(ctx, updatedJob) + By("Checking that the queue update fails") + Expect(err).To(HaveOccurred(), "Expected an error when updating the queue, but update succeeded") + Expect(err).To(MatchError(ContainSubstring("spec.runPolicy.schedulingPolicy.queue is immutable"), "The error message did not contain the expected message")) + return err != nil + }, testutil.Timeout, testutil.Interval).Should(BeTrue()) By("Validating the queue was not updated") freshJob := &kubeflowv1.PyTorchJob{}