wip

check_logs.py

@@ -5,8 +5,10 @@
 import copy
 import csv
 import datetime
+import itertools
 import json
 import logging
+from operator import itemgetter
 import os.path
 import re
 import subprocess
@@ -223,30 +225,92 @@ def get_statuses(gh, org, repo, pr_num):
 # * error - the error line from the log
 # * url - link to ansible log file
 # * context_lines - log lines to give some context for the error
+# def get_errors_from_ansible_log(args, log_url):
+#     errors = []
+#     task_lines = []
+#     error_lines = []
+#     in_task = False
+#     logging.debug("Getting errors from ansible log [%s]", log_url)
+#     for line in get_file_data(args, log_url):
+#         if line.startswith("PLAY") or line.startswith("TASK ["):
+#             # we are possibly at the end of a task - see if there are any errors
+#             # we need to report
+#             if in_task and task_lines and error_lines:
+#                 error = {"url": log_url, "context_lines": copy.deepcopy(task_lines), "error": copy.deepcopy(error_lines)}
+#                 errors.append(error)
+#                 in_task = False
+#             error_lines = []
+#             if line.startswith("TASK ["):
+#                 # we're at the beginning of a new task - reset
+#                 in_task = True
+#                 task_lines = [line]
+#         elif in_task:
+#             task_lines.append(line)
+#             if line.startswith("fatal:"):
+#                 error_lines.append(line)
+#     logging.debug("Found [%d] errors", len(errors))
+#     return errors
+
 def get_errors_from_ansible_log(args, log_url):
     errors = []
+    pattern = r"ANSIBLE-(\d+\.\d+)"
+    ansible_version_matches = re.findall(pattern, log_url)
+    ansible_version = ansible_version_matches[0] if ansible_version_matches else "UNKNOWN"
+    current_task = None
+    total_failed = 0
     task_lines = []
-    error_lines = []
-    in_task = False
+    task_has_fatal = False
+    task_path = None
+
     logging.debug("Getting errors from ansible log [%s]", log_url)
+
+    # Extracts the system role name
+    extracted_part = log_url.split("logs/")[1]
+    start_index = extracted_part.index("tf_") + 3  # +3 to skip "tf_"
+    end_index = extracted_part.index("-", start_index)
+    role = extracted_part[start_index:end_index]
+
     for line in get_file_data(args, log_url):
-        if line.startswith("PLAY") or line.startswith("TASK ["):
-            # we are possibly at the end of a task - see if there are any errors
-            # we need to report
-            if in_task and task_lines and error_lines:
-                error = {"url": log_url, "context_lines": copy.deepcopy(task_lines), "error": copy.deepcopy(error_lines)}
+        if line.startswith("TASK ") or line.startswith("PLAY ") or line.startswith("META "):
+            # end of current task and possibly start of new task
+            if task_lines and task_has_fatal:
+                # Extract task name from the first task line
+                task_match = re.search(r"TASK\s\[(.*?)\]", task_lines[0])
+                if task_match:
+                    current_task = task_match.group(1)
+                # end task
+                error = {
+                    "Url": log_url,
+                    "Role": role,
+                    "Ansible Version": ansible_version,
+                    "Task": current_task,
+                    "Detail": copy.deepcopy(task_lines[3:]),
+                    "Task Path": task_path,
+                }
                 errors.append(error)
-                in_task = False
-            error_lines = []
-            if line.startswith("TASK ["):
-                # we're at the beginning of a new task - reset
-                in_task = True
-                task_lines = [line]
-        elif in_task:
-            task_lines.append(line)
+            if line.startswith("TASK "):
+                task_lines = [line.strip()]
+            else:
+                task_lines = []
+            task_has_fatal = False
+            task_path = None
+        elif task_lines:
+            task_lines.append(line.strip())
             if line.startswith("fatal:"):
-                error_lines.append(line)
-    logging.debug("Found [%d] errors", len(errors))
+                task_has_fatal = True
+            elif line.startswith("task path:"):
+                task_path_match = re.search(r"task path: (.*)", line)
+                if task_path_match:
+                    task_path = task_path_match.group(1)
+        else:
+            match = re.search(r"\sfailed=(\d+)\s", line)
+            if match:
+                total_failed += int(match.group(1))
+
+    logging.debug("Found [%d] errors and Ansible reported [%d] failures", len(errors), total_failed)
+    for error in errors:
+        error["Fails expected"] = total_failed
+
     return errors
 
 
@@ -606,6 +670,9 @@ def parse_ansible_junit_log(log_file):
 
 
 def print_ansible_errors(args, errors):
+    if not errors:
+        print("No errors found")
+        return
     headings = list(errors[0].keys())
     if args.csv_errors:
         if args.csv_errors == "-":
@@ -630,13 +697,24 @@ def print_ansible_errors(args, errors):
         else:
             sh = gc.open_by_url(args.gspread)
         values = [headings]
+        current_value = {}
+        if args.group_by:
+            errors = sorted(errors, key=itemgetter(*args.group_by))
+            for key in args.group_by:
+                current_value[key] = None
         for error in errors:
             value_list = []
             for key in headings:
-                if isinstance(error[key], list):
-                    value = "\n".join(error[key])
+                item = error[key]
+                if key in current_value:
+                    if item == current_value.get(key):
+                        item = ""
+                    else:
+                        current_value[key] = item
+                if isinstance(item, list):
+                    value = "\n".join(item)
                 else:
-                    value = str(error[key])
+                    value = str(item)
                 value_list.append(value)
             values.append(value_list)
         sh.values_update("sheet1", {"valueInputOption": "USER_ENTERED"}, {"values": values})
@@ -743,6 +821,12 @@ def main():
         default=os.environ["HOME"] + "/.config/gspread/google_secret.json",
         help="path to google spreadsheet api credentials",
     )
+    parser.add_argument(
+        "--group-by",
+        default=[],
+        action="append",
+        help="For gspread - group items by these columns",
+    )
 
     args = parser.parse_args()