From 31f008d46692a3867704763ca877abcfd8d3e9b8 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Fri, 23 Jun 2023 14:22:41 -0700 Subject: [PATCH 01/16] update flow of wait timeout --- manifest/provider/apply.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index e432fc02f5..91c64e7e40 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -440,7 +440,7 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot if err == context.DeadlineExceeded { resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{ - Severity: tfprotov5.DiagnosticSeverityError, + Severity: tfprotov5.DiagnosticSeverityWarning, Summary: "Operation timed out", Detail: "Terraform timed out waiting on the operation to complete", }) @@ -451,8 +451,8 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot Summary: "Error waiting for operation to complete", Detail: err.Error(), }) + return resp, nil } - return resp, nil } } From 1e9cebe1f31a23d10b9ef73fad5b4530ebb35fd5 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Tue, 27 Jun 2023 10:49:41 -0700 Subject: [PATCH 02/16] show waitconfig when operation times out --- manifest/provider/apply.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index 91c64e7e40..ad0163ead9 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -442,7 +442,7 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot &tfprotov5.Diagnostic{ Severity: tfprotov5.DiagnosticSeverityWarning, Summary: "Operation timed out", - Detail: "Terraform timed out waiting on the operation to complete", + Detail: fmt.Sprintf("Resource created but timed out while waiting for %v", waitConfig), }) } else { resp.Diagnostics = append(resp.Diagnostics, From 0574da37af7f4e6538c0062ceb09b4535d92eb44 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Wed, 28 Jun 2023 19:47:04 -0700 Subject: [PATCH 03/16] updated error output --- manifest/provider/apply.go | 4 ++-- manifest/provider/waiter.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index ad0163ead9..dfa14e95e0 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -437,12 +437,12 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot if !waitConfig.IsNull() { err = s.waitForCompletion(ctxDeadline, waitConfig, rs, rname, wt, th) if err != nil { - if err == context.DeadlineExceeded { + if err == fmt.Errorf("timed out waiting on conditions") { resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{ Severity: tfprotov5.DiagnosticSeverityWarning, Summary: "Operation timed out", - Detail: fmt.Sprintf("Resource created but timed out while waiting for %v", waitConfig), + Detail: "Terraform timed out waiting on the operation to complete", }) } else { resp.Diagnostics = append(resp.Diagnostics, diff --git a/manifest/provider/waiter.go b/manifest/provider/waiter.go index e4e15729bd..8402e80f44 100644 --- a/manifest/provider/waiter.go +++ b/manifest/provider/waiter.go @@ -145,7 +145,7 @@ func (w *FieldWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return context.DeadlineExceeded + return fmt.Errorf("timed out waiting on conditions") } } From ed407a96e03417061ba953583e62d84b28559bf7 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Wed, 19 Jul 2023 12:06:39 -0700 Subject: [PATCH 04/16] include other wait conditions --- manifest/provider/apply.go | 2 +- manifest/provider/waiter.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index dfa14e95e0..e1c8679663 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -437,7 +437,7 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot if !waitConfig.IsNull() { err = s.waitForCompletion(ctxDeadline, waitConfig, rs, rname, wt, th) if err != nil { - if err == fmt.Errorf("timed out waiting on conditions") { + if err == fmt.Errorf("timed out waiting on conditions") || err == fmt.Errorf("timed out waiting on configured conditions to be met") || err == fmt.Errorf("timed out waiting on rollout to complete") { resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{ Severity: tfprotov5.DiagnosticSeverityWarning, diff --git a/manifest/provider/waiter.go b/manifest/provider/waiter.go index 8402e80f44..74bc40077f 100644 --- a/manifest/provider/waiter.go +++ b/manifest/provider/waiter.go @@ -283,7 +283,7 @@ func (w *RolloutWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return context.DeadlineExceeded + return fmt.Errorf("timed out waiting on rollout to complete") } } @@ -333,7 +333,7 @@ func (w *ConditionsWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return context.DeadlineExceeded + return fmt.Errorf("timed out waiting on configured conditions to be met") } } From 1bdced5d36ab23024e3750b2fa501b0c05cb1f01 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Thu, 20 Jul 2023 12:11:57 -0700 Subject: [PATCH 05/16] format conditions using WaiterError type --- manifest/provider/apply.go | 4 ++-- manifest/provider/waiter.go | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index e1c8679663..5d3ac3bb6b 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -437,12 +437,12 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot if !waitConfig.IsNull() { err = s.waitForCompletion(ctxDeadline, waitConfig, rs, rname, wt, th) if err != nil { - if err == fmt.Errorf("timed out waiting on conditions") || err == fmt.Errorf("timed out waiting on configured conditions to be met") || err == fmt.Errorf("timed out waiting on rollout to complete") { + if reason, ok := err.(WaiterError); ok { resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{ Severity: tfprotov5.DiagnosticSeverityWarning, Summary: "Operation timed out", - Detail: "Terraform timed out waiting on the operation to complete", + Detail: reason.Error(), }) } else { resp.Diagnostics = append(resp.Diagnostics, diff --git a/manifest/provider/waiter.go b/manifest/provider/waiter.go index 74bc40077f..e7ffa4f0d3 100644 --- a/manifest/provider/waiter.go +++ b/manifest/provider/waiter.go @@ -42,6 +42,14 @@ type Waiter interface { Wait(context.Context) error } +type WaiterError struct { + Reason string +} + +func (e WaiterError) Error() string { + return fmt.Sprintf("timed out waiting on %v", e.Reason) +} + // NewResourceWaiter constructs an appropriate Waiter using the supplied waitForBlock configuration func NewResourceWaiter(resource dynamic.ResourceInterface, resourceName string, resourceType tftypes.Type, th map[string]string, waitForBlock tftypes.Value, hl hclog.Logger) (Waiter, error) { var waitForBlockVal map[string]tftypes.Value @@ -145,7 +153,7 @@ func (w *FieldWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return fmt.Errorf("timed out waiting on conditions") + return WaiterError{Reason: "FieldsMatchers"} } } @@ -283,7 +291,7 @@ func (w *RolloutWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return fmt.Errorf("timed out waiting on rollout to complete") + return WaiterError{Reason: "rollout to complete"} } } @@ -333,7 +341,7 @@ func (w *ConditionsWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return fmt.Errorf("timed out waiting on configured conditions to be met") + return WaiterError{Reason: "conditions"} } } From 88d7be14dfa0808fb020abf94eaa12d93c3a7718 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Thu, 20 Jul 2023 13:44:37 -0700 Subject: [PATCH 06/16] update wait_test string check --- manifest/test/acceptance/wait_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index 7c652aefdb..fc9a48d3c7 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -219,7 +219,7 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { tf.Init(ctx) err = tf.Apply(ctx) - if err == nil || !strings.Contains(err.Error(), "Terraform timed out waiting on the operation to complete") { + if err == nil || !strings.Contains(err.Error(), ""timed out waiting on"") { t.Fatalf("Waiter should have timed out") } } From 27e94ec806e8e5c0315fffded5eda09108ad5102 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Thu, 20 Jul 2023 13:45:12 -0700 Subject: [PATCH 07/16] typo --- manifest/test/acceptance/wait_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index fc9a48d3c7..27e1f071b7 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -219,7 +219,7 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { tf.Init(ctx) err = tf.Apply(ctx) - if err == nil || !strings.Contains(err.Error(), ""timed out waiting on"") { + if err == nil || !strings.Contains(err.Error(), "timed out waiting on") { t.Fatalf("Waiter should have timed out") } } From df4d033d2b57f6411fd1b276989821bd3760c23d Mon Sep 17 00:00:00 2001 From: BBBmau Date: Thu, 20 Jul 2023 16:35:56 -0700 Subject: [PATCH 08/16] add context exceeded case --- manifest/provider/apply.go | 1 - manifest/test/acceptance/wait_test.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index 779d6d5b23..eda665ec1d 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -452,7 +452,6 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot Summary: fmt.Sprintf(`Failed to read resource %q after wait conditions`, rname), Detail: err.Error(), }) - return resp, nil } result = r diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index 27e1f071b7..029cad7b82 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -219,7 +219,8 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { tf.Init(ctx) err = tf.Apply(ctx) - if err == nil || !strings.Contains(err.Error(), "timed out waiting on") { + //panic(err.Error()) + if err == nil || (!strings.Contains(err.Error(), "timed out waiting on") && !strings.Contains(err.Error(), "client rate limiter")) { t.Fatalf("Waiter should have timed out") } } From 03f6d4328c414a8eced410be5834b5be82adc499 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Fri, 21 Jul 2023 09:49:29 -0700 Subject: [PATCH 09/16] remove extra condition --- manifest/test/acceptance/wait_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index 029cad7b82..8fb725ce8b 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -220,7 +220,7 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { err = tf.Apply(ctx) //panic(err.Error()) - if err == nil || (!strings.Contains(err.Error(), "timed out waiting on") && !strings.Contains(err.Error(), "client rate limiter")) { + if err == nil || !strings.Contains(err.Error(), "timed out waiting on") { t.Fatalf("Waiter should have timed out") } } From 40500ef5331ee9223993ff3f0b3dc087d06aae80 Mon Sep 17 00:00:00 2001 From: John Houston Date: Fri, 21 Jul 2023 13:39:32 -0400 Subject: [PATCH 10/16] remove unneeded comments from acctest --- manifest/test/acceptance/wait_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index 8fb725ce8b..27e1f071b7 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -219,7 +219,6 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { tf.Init(ctx) err = tf.Apply(ctx) - //panic(err.Error()) if err == nil || !strings.Contains(err.Error(), "timed out waiting on") { t.Fatalf("Waiter should have timed out") } From 3b026f6e80147420005681eea75074420638e0b0 Mon Sep 17 00:00:00 2001 From: John Houston Date: Fri, 21 Jul 2023 13:39:52 -0400 Subject: [PATCH 11/16] use correct context to get state when waiter times out --- manifest/provider/apply.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/manifest/provider/apply.go b/manifest/provider/apply.go index eda665ec1d..22451dc5ed 100644 --- a/manifest/provider/apply.go +++ b/manifest/provider/apply.go @@ -428,7 +428,7 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot if reason, ok := err.(WaiterError); ok { resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{ - Severity: tfprotov5.DiagnosticSeverityWarning, + Severity: tfprotov5.DiagnosticSeverityError, Summary: "Operation timed out", Detail: reason.Error(), }) @@ -442,8 +442,7 @@ func (s *RawProviderServer) ApplyResourceChange(ctx context.Context, req *tfprot return resp, nil } } - - r, err := rs.Get(ctxDeadline, rname, metav1.GetOptions{}) + r, err := rs.Get(ctx, rname, metav1.GetOptions{}) if err != nil { s.logger.Error("[ApplyResourceChange][ReadAfterWait]", "API error", dump(err), "API response", dump(result)) resp.Diagnostics = append(resp.Diagnostics, From 0dd1d38bc257529c3780e065e08e71f78cdbad13 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Tue, 25 Jul 2023 09:25:04 -0700 Subject: [PATCH 12/16] fix reason message --- manifest/provider/waiter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest/provider/waiter.go b/manifest/provider/waiter.go index e7ffa4f0d3..43a6e122d9 100644 --- a/manifest/provider/waiter.go +++ b/manifest/provider/waiter.go @@ -153,7 +153,7 @@ func (w *FieldWaiter) Wait(ctx context.Context) error { for { if deadline, ok := ctx.Deadline(); ok { if time.Now().After(deadline) { - return WaiterError{Reason: "FieldsMatchers"} + return WaiterError{Reason: "field matchers"} } } From fef8dfc2bb8973edb065fc7020f63abaf6bbd28a Mon Sep 17 00:00:00 2001 From: BBBmau Date: Wed, 26 Jul 2023 13:29:49 -0700 Subject: [PATCH 13/16] add resourceCheck after waitCondition --- manifest/test/acceptance/wait_test.go | 9 +++++++++ manifest/test/helper/state/state_helper.go | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index 27e1f071b7..9ed5745173 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -222,6 +222,15 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "timed out waiting on") { t.Fatalf("Waiter should have timed out") } + + st, err := tf.State(ctx) + if err != nil { + t.Fatalf("Failed to get state: %q", err) + } + tfstate := tfstatehelper.NewHelper(st) + if !tfstate.ResourceExists(t, "kubernetes_manifest.test") { + t.Fatalf("Expected resource does not exist.") + } } func TestKubernetesManifest_WaitFields_Annotations(t *testing.T) { diff --git a/manifest/test/helper/state/state_helper.go b/manifest/test/helper/state/state_helper.go index 2d38d1c64d..430b49b433 100644 --- a/manifest/test/helper/state/state_helper.go +++ b/manifest/test/helper/state/state_helper.go @@ -26,6 +26,14 @@ type Helper struct { func NewHelper(tfstate *tfjson.State) *Helper { return &Helper{tfstate} } +func (s *Helper) ResourceExists(t *testing.T, resourceAddress string) bool { + t.Helper() + _, err := getAttributesValuesFromResource(s, resourceAddress) + if err != nil { + return false + } + return true +} // getAttributesValuesFromResource pulls out the AttributeValues field from the resource at the given address func getAttributesValuesFromResource(state *Helper, address string) (interface{}, error) { From ecee1e11651e1020bd99c29df9aa4448bf5e3340 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Wed, 26 Jul 2023 13:41:48 -0700 Subject: [PATCH 14/16] add changelog-entry --- .changelog/2163.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/2163.txt diff --git a/.changelog/2163.txt b/.changelog/2163.txt new file mode 100644 index 0000000000..1bf1ea3bf7 --- /dev/null +++ b/.changelog/2163.txt @@ -0,0 +1,3 @@ +```release-note:bug +`manifest/provider/apply.go`: update flow of wait timeout +``` \ No newline at end of file From 2300a684f0f24152f33b40b536d1136defdfb2ad Mon Sep 17 00:00:00 2001 From: BBBmau Date: Wed, 26 Jul 2023 17:38:53 -0700 Subject: [PATCH 15/16] refactor --- manifest/test/acceptance/wait_test.go | 2 +- manifest/test/helper/state/state_helper.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/manifest/test/acceptance/wait_test.go b/manifest/test/acceptance/wait_test.go index 9ed5745173..6eb7cd916d 100644 --- a/manifest/test/acceptance/wait_test.go +++ b/manifest/test/acceptance/wait_test.go @@ -229,7 +229,7 @@ func TestKubernetesManifest_Wait_InvalidCondition(t *testing.T) { } tfstate := tfstatehelper.NewHelper(st) if !tfstate.ResourceExists(t, "kubernetes_manifest.test") { - t.Fatalf("Expected resource does not exist.") + t.Fatalf("Expected resource to exist in state") } } diff --git a/manifest/test/helper/state/state_helper.go b/manifest/test/helper/state/state_helper.go index 430b49b433..7a2d6b9e1d 100644 --- a/manifest/test/helper/state/state_helper.go +++ b/manifest/test/helper/state/state_helper.go @@ -29,10 +29,7 @@ func NewHelper(tfstate *tfjson.State) *Helper { func (s *Helper) ResourceExists(t *testing.T, resourceAddress string) bool { t.Helper() _, err := getAttributesValuesFromResource(s, resourceAddress) - if err != nil { - return false - } - return true + return err == nil } // getAttributesValuesFromResource pulls out the AttributeValues field from the resource at the given address From cbd3b104015127638ed3184bf31a906c9aa4612b Mon Sep 17 00:00:00 2001 From: BBBmau Date: Thu, 27 Jul 2023 10:09:10 -0700 Subject: [PATCH 16/16] update changelog-entry --- .changelog/2163.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/2163.txt b/.changelog/2163.txt index 1bf1ea3bf7..7cfed89cd5 100644 --- a/.changelog/2163.txt +++ b/.changelog/2163.txt @@ -1,3 +1,3 @@ ```release-note:bug -`manifest/provider/apply.go`: update flow of wait timeout +`manifest/provider/apply.go`: update flow in `wait` block to fix timeout bug within tf apply where the resource is created and appears in Kubernetes but does not appear in TF state file after deadline. The fix would ensure that the resource has been created in the state file while also tainting the resource requiring the user to make the necessary changes in order for their to not be another timeout error. ``` \ No newline at end of file