[Wf-Diagnostics] Point to activity task for activity failures (cadenc…

…e-workflow#6355)
sankari165 · Oct 14, 2024 · a32159c · a32159c
1 parent 0a52b25
commit a32159c
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 14 deletions.
diff --git a/service/worker/diagnostics/invariant/failure/failure.go b/service/worker/diagnostics/invariant/failure/failure.go
@@ -67,10 +67,16 @@ func (f *failure) Check(context.Context) ([]invariant.InvariantCheckResult, erro
 		if event.GetActivityTaskFailedEventAttributes() != nil && event.ActivityTaskFailedEventAttributes.Reason != nil {
 			attr := event.ActivityTaskFailedEventAttributes
 			reason := attr.Reason
+			scheduled := fetchScheduledEvent(attr, events)
+			started := fetchStartedEvent(attr, events)
 			result = append(result, invariant.InvariantCheckResult{
 				InvariantType: ActivityFailed.String(),
 				Reason:        errorTypeFromReason(*reason).String(),
-				Metadata:      invariant.MarshalData(failureMetadata{Identity: attr.Identity}),
+				Metadata: invariant.MarshalData(failureMetadata{
+					Identity:          attr.Identity,
+					ActivityScheduled: scheduled,
+					ActivityStarted:   started,
+				}),
 			})
 		}
 	}
@@ -99,6 +105,24 @@ func fetchIdentity(attr *types.WorkflowExecutionFailedEventAttributes, events []
 	return ""
 }
 
+func fetchScheduledEvent(attr *types.ActivityTaskFailedEventAttributes, events []*types.HistoryEvent) *types.ActivityTaskScheduledEventAttributes {
+	for _, event := range events {
+		if event.ID == attr.GetScheduledEventID() {
+			return event.GetActivityTaskScheduledEventAttributes()
+		}
+	}
+	return nil
+}
+
+func fetchStartedEvent(attr *types.ActivityTaskFailedEventAttributes, events []*types.HistoryEvent) *types.ActivityTaskStartedEventAttributes {
+	for _, event := range events {
+		if event.ID == attr.GetStartedEventID() {
+			return event.GetActivityTaskStartedEventAttributes()
+		}
+	}
+	return nil
+}
+
 func (f *failure) RootCause(ctx context.Context, issues []invariant.InvariantCheckResult) ([]invariant.InvariantRootCauseResult, error) {
 	result := make([]invariant.InvariantRootCauseResult, 0)
 	for _, issue := range issues {

diff --git a/service/worker/diagnostics/invariant/failure/failure_test.go b/service/worker/diagnostics/invariant/failure/failure_test.go
@@ -44,6 +44,19 @@ func Test__Check(t *testing.T) {
 	}
 	metadataInBytes, err := json.Marshal(metadata)
 	require.NoError(t, err)
+	actMetadata := failureMetadata{
+		Identity: "localhost",
+		ActivityScheduled: &types.ActivityTaskScheduledEventAttributes{
+			ActivityID:   "101",
+			ActivityType: &types.ActivityType{Name: "test-activity"},
+		},
+		ActivityStarted: &types.ActivityTaskStartedEventAttributes{
+			Identity: "localhost",
+			Attempt:  0,
+		},
+	}
+	actMetadataInBytes, err := json.Marshal(actMetadata)
+	require.NoError(t, err)
 	testCases := []struct {
 		name           string
 		testData       *types.GetWorkflowExecutionHistoryResponse
@@ -57,17 +70,17 @@ func Test__Check(t *testing.T) {
 				{
 					InvariantType: ActivityFailed.String(),
 					Reason:        GenericError.String(),
-					Metadata:      metadataInBytes,
+					Metadata:      actMetadataInBytes,
 				},
 				{
 					InvariantType: ActivityFailed.String(),
 					Reason:        PanicError.String(),
-					Metadata:      metadataInBytes,
+					Metadata:      actMetadataInBytes,
 				},
 				{
 					InvariantType: ActivityFailed.String(),
 					Reason:        CustomError.String(),
-					Metadata:      metadataInBytes,
+					Metadata:      actMetadataInBytes,
 				},
 				{
 					InvariantType: WorkflowFailed.String(),
@@ -95,24 +108,44 @@ func failedWfHistory() *types.GetWorkflowExecutionHistoryResponse {
 		History: &types.History{
 			Events: []*types.HistoryEvent{
 				{
-					ActivityTaskFailedEventAttributes: &types.ActivityTaskFailedEventAttributes{
-						Reason:   common.StringPtr("cadenceInternal:Generic"),
-						Details:  []byte("test-activity-failure"),
+					ID: 1,
+					ActivityTaskScheduledEventAttributes: &types.ActivityTaskScheduledEventAttributes{
+						ActivityID:   "101",
+						ActivityType: &types.ActivityType{Name: "test-activity"},
+					},
+				},
+				{
+					ID: 2,
+					ActivityTaskStartedEventAttributes: &types.ActivityTaskStartedEventAttributes{
 						Identity: "localhost",
+						Attempt:  0,
 					},
 				},
 				{
 					ActivityTaskFailedEventAttributes: &types.ActivityTaskFailedEventAttributes{
-						Reason:   common.StringPtr("cadenceInternal:Panic"),
-						Details:  []byte("test-activity-failure"),
-						Identity: "localhost",
+						Reason:           common.StringPtr("cadenceInternal:Generic"),
+						Details:          []byte("test-activity-failure"),
+						Identity:         "localhost",
+						ScheduledEventID: 1,
+						StartedEventID:   2,
 					},
 				},
 				{
 					ActivityTaskFailedEventAttributes: &types.ActivityTaskFailedEventAttributes{
-						Reason:   common.StringPtr("custom error"),
-						Details:  []byte("test-activity-failure"),
-						Identity: "localhost",
+						Reason:           common.StringPtr("cadenceInternal:Panic"),
+						Details:          []byte("test-activity-failure"),
+						Identity:         "localhost",
+						ScheduledEventID: 1,
+						StartedEventID:   2,
+					},
+				},
+				{
+					ActivityTaskFailedEventAttributes: &types.ActivityTaskFailedEventAttributes{
+						Reason:           common.StringPtr("custom error"),
+						Details:          []byte("test-activity-failure"),
+						Identity:         "localhost",
+						ScheduledEventID: 1,
+						StartedEventID:   2,
 					},
 				},
 				{

diff --git a/service/worker/diagnostics/invariant/failure/types.go b/service/worker/diagnostics/invariant/failure/types.go
@@ -22,6 +22,8 @@
 
 package failure
 
+import "github.com/uber/cadence/common/types"
+
 type ErrorType string
 
 const (
@@ -47,5 +49,7 @@ func (f FailureType) String() string {
 }
 
 type failureMetadata struct {
-	Identity string
+	Identity          string
+	ActivityScheduled *types.ActivityTaskScheduledEventAttributes
+	ActivityStarted   *types.ActivityTaskStartedEventAttributes
 }