From 00a284364906ced4d71e5109dec8f7f2f3cae076 Mon Sep 17 00:00:00 2001 From: Alan Ma Date: Thu, 6 Jun 2024 20:23:32 -0700 Subject: [PATCH] Include fatal reason for pod pending events (#39924) --- .../kubernetes/executors/kubernetes_executor_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py index b8235bb5ac49b..710d5d1f9b497 100644 --- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py @@ -230,7 +230,6 @@ def process_status( self.kube_config.worker_pod_pending_fatal_container_state_reasons and "status" in event["raw_object"] ): - self.log.info("Event: %s Pending, annotations: %s", pod_name, annotations_string) # Init containers and base container statuses to check. # Skipping the other containers statuses check. container_statuses_to_check = [] @@ -250,10 +249,18 @@ def process_status( and container_status_state["waiting"]["message"] == "pull QPS exceeded" ): continue + self.log.error( + "Event: %s has container %s with fatal reason %s", + pod_name, + container_status["name"], + container_status_state["waiting"]["reason"], + ) self.watcher_queue.put( (pod_name, namespace, TaskInstanceState.FAILED, annotations, resource_version) ) break + else: + self.log.info("Event: %s Pending, annotations: %s", pod_name, annotations_string) else: self.log.debug("Event: %s Pending, annotations: %s", pod_name, annotations_string) elif status == "Failed":