Fix race condition issue

Makefile

@@ -68,7 +68,7 @@ generate-code:
 	./hack/update-gencode.sh
 
 unit-test:
-	go list ./... | grep -v e2e | xargs go test -v -cover -covermode atomic -coverprofile coverage.txt
+	go list ./... | grep -v e2e | xargs go test -v -cover -covermode atomic -coverprofile coverage.txt -race
 
 e2e-test-kind:
 	./hack/run-e2e-kind.sh

pkg/controllers/job/job_controller_actions.go

@@ -253,6 +253,7 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt
 
 	waitCreationGroup := sync.WaitGroup{}
 	waitCreationGroup.Add(len(podToCreate))
+	stateMutex := sync.Mutex{}
 	for _, pod := range podToCreate {
 		go func(pod *v1.Pod) {
 			defer waitCreationGroup.Done()
@@ -263,13 +264,17 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt
 				// So gang-scheduling could schedule the Job successfully
 				glog.Errorf("Failed to create pod %s for Job %s, err %#v",
 					pod.Name, job.Name, err)
+				stateMutex.Lock()
 				creationErrs = append(creationErrs, fmt.Errorf("failed to create pod %s, err: %#v", pod.Name, err))
+				stateMutex.Unlock()
 			} else {
 				if err != nil && apierrors.IsAlreadyExists(err) {
 					cc.resyncTask(pod)
 				}
 
+				stateMutex.Lock()
 				classifyAndAddUpPodBaseOnPhase(newPod, &pending, &running, &succeeded, &failed, &unknown)
+				stateMutex.Unlock()
 				glog.V(3).Infof("Created Task <%s> of Job <%s/%s>",
 					pod.Name, job.Namespace, job.Name)
 			}
@@ -297,12 +302,16 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt
 				// So gang-scheduling could schedule the Job successfully
 				glog.Errorf("Failed to delete pod %s for Job %s, err %#v",
 					pod.Name, job.Name, err)
+				stateMutex.Lock()
 				deletionErrs = append(deletionErrs, err)
+				stateMutex.Unlock()
 				cc.resyncTask(pod)
 			} else {
 				glog.V(3).Infof("Deleted Task <%s> of Job <%s/%s>",
 					pod.Name, job.Namespace, job.Name)
+				stateMutex.Lock()
 				terminating++
+				stateMutex.Unlock()
 			}
 		}(pod)
 	}