get errors from ansible logs from beaker tests

check_logs.py

@@ -253,9 +253,6 @@ def get_statuses(gh, org, repo, pr_num):
 
 def get_errors_from_ansible_log(args, log_url):
     errors = []
-    pattern = r"ANSIBLE-(\d+\.\d+)"
-    ansible_version_matches = re.findall(pattern, log_url)
-    ansible_version = ansible_version_matches[0] if ansible_version_matches else "UNKNOWN"
     current_task = None
     total_failed = 0
     task_lines = []
@@ -266,9 +263,19 @@ def get_errors_from_ansible_log(args, log_url):
 
     # Extracts the system role name
     extracted_part = log_url.split("logs/")[1]
-    start_index = extracted_part.index("tf_") + 3  # +3 to skip "tf_"
-    end_index = extracted_part.index("-", start_index)
-    role = extracted_part[start_index:end_index]
+    start_index = extracted_part.find("tf_") + 3  # +3 to skip "tf_"
+    if start_index > 2:
+        end_index = extracted_part.index("-", start_index)
+        role = extracted_part[start_index:end_index]
+        pattern = r"ANSIBLE-(\d+\.\d+)"
+        ansible_version_matches = re.findall(pattern, log_url)
+        ansible_version = ansible_version_matches[0] if ansible_version_matches else "UNKNOWN"
+    else:
+        # https://....//SYSTEM-ROLE-$ROLENAME_$TEST_NAME.yml-legacy-ANSIBLE-2.log
+        match = re.search(r'/SYSTEM-ROLE-([a-z0-9_]+)_(tests_[a-z0-9_]+[.]yml)-.*-ANSIBLE-([0-9.]+)[.]log$', log_url)
+        role = match.group(1)
+        test = match.group(2)  # unused for now
+        ansible_version = match.group(3)
 
     for line in get_file_data(args, log_url):
         if line.startswith("TASK ") or line.startswith("PLAY ") or line.startswith("META "):
@@ -509,7 +516,7 @@ def get_beaker_job_info(args, job):
     data["post_install_end"] = bs.find("installation").get("postinstall_finished")
     data["tasks"] = []
     for task in bs.find_all("task"):
-        task_data = {}
+        task_data = {"errors": []}
         for key in (
             "name",
             "result",
@@ -531,6 +538,7 @@ def get_beaker_job_info(args, job):
                 if name == "taskout.log":
                     task_data["job_data"] = parse_beaker_job_log(args, task_data["start_time"], link)
                 elif name.startswith("SYSTEM-ROLE-"):
+                    task_data["errors"].extend(get_errors_from_ansible_log(args, link))
                     log_urls.append(link)
             task_data["logs"] = log_urls
             role = None
@@ -554,6 +562,7 @@ def print_beaker_job_info(args, info):
         f"Distro [{info['distro']}] arch [{info['arch']}] whiteboard [{info['whiteboard']}] job [{info['job']}]"
     )
     print(f"  Install start [{info['install_start']}] end [{info['post_install_end']}]")
+    errors = []
     for task in info["tasks"]:
         sys.stdout.write("  Task")
         for key in (
@@ -578,11 +587,13 @@ def print_beaker_job_info(args, info):
         )
         if job_data["status"] != "RUNNING" and job_data.get("duration"):
             print(f"  Duration {job_data['duration']}")
+        errors.extend(task["errors"])
         if args.failed_tests_to_show > 0:
             for failed_test in job_data["failed"][-args.failed_tests_to_show:]:  # fmt: skip
                 print(f"    failed {failed_test}")
             print_avcs_and_tasks(args, task)
     print("")
+    print_ansible_errors(args, errors)
 
 
 def get_logs_from_beaker(args):
@@ -694,8 +705,11 @@ def print_ansible_errors(args, errors):
             sh = gc.create("LSR testing spreadsheet")
             # need to show url to new spreadsheet
             # and share with given user
+            worksheet = sh.get_worksheet(0)
         else:
             sh = gc.open_by_url(args.gspread)
+            worksheet = sh.get_worksheet(0)
+            worksheet.clear()
         values = [headings]
         current_value = {}
         if args.group_by:
@@ -715,9 +729,11 @@ def print_ansible_errors(args, errors):
                     value = "\n".join(item)
                 else:
                     value = str(item)
+                if len(value) > 20000:
+                    value = value[:20000] + ".... truncated"
                 value_list.append(value)
             values.append(value_list)
-        sh.values_update("sheet1", {"valueInputOption": "USER_ENTERED"}, {"values": values})
+        sh.values_update(worksheet.title, {"valueInputOption": "USER_ENTERED"}, {"values": values})
 
 
 def main():