check jenkins fixes

README.md

@@ -940,6 +940,45 @@ actions:
 ```
 which isn't very useful unless you are debugging the script and/or Jenkins itself.
 
+### print_task_tests_info
+
+Use this to see information about the task, including the individual tests
+statuses.
+```
+> ./check_jenkins.py print_task_tests_info 12345
+Role:nbde_client PR:80 Platform:RHEL-8.8.0-20220921.0 Arch:x86_64
+Node:production-3 IP:10.0.0.3 Workspace:/var/lib/jenkins/workspace/ci-test-jobname@6
+Stage                State      Result     GuestID                              Test Workdir
+COMPLETE             OK         PASSED     c3767ea3-934c-408d-b42f-d7639e3e30ca tests_bind_high_availability.yml work-tests_bind_high_availability.ymlrHBR08
+COMPLETE             OK         PASSED     06e43cbc-0090-41ad-8477-16fe0ff43447 tests_default.yml work-tests_default.yml16nMco
+COMPLETE             OK         PASSED     d05aaf71-5075-47a1-b50a-9904cc33b699 tests_default_vars.yml work-tests_default_vars.ymlQdWTmI
+GUEST_PROVISIONING   OK         UNDEFINED  unknown                              tests_include_vars_from_parent.yml unknown
+GUEST_PROVISIONING   OK         UNDEFINED  unknown                              tests_key_rotation.yml unknown
+GUEST_PROVISIONING   OK         UNDEFINED  unknown                              tests_passphrase_temporary.yml unknown
+CREATED              OK         UNDEFINED  unknown                              tests_passphrase_temporary_keyfile.yml unknown
+```
+`Node` is the internal node name used by Jenkins.  `Workspace` is the path to
+the directory on the node which is used to hold the test artifacts before they
+are published.  For example, if you want to see the ansible results for the
+completed tests_bind_high_availability.yml test:
+```
+ssh -i /path/to/key [email protected]
+cd /var/lib/jenkins/workspace/ci-test-jobname@6
+# from here you can see citool-debug.txt, etc.
+cat work-tests_bind_high_availability.ymlrHBR08/ansible-output.txt
+cat guest-setup-c3767ea3-934c-408d-b42f-d7639e3e30ca/pre-installation-artifact-workaround.txt
+etc.
+```
+
+### print_task_console
+
+If you just want to see the task output without having to use the web browser:
+```
+./check_jenkins.py print_task_console 12345 30
+... bunch of text here ...
+```
+This will show the last 30 lines of the console.
+
 # configure_squid
 
 The `configure_squid` directory stores the playbook that you can use to

check_jenkins.py

@@ -18,6 +18,8 @@
 url = cfg[cfg["current"]]["url"]
 username = cfg[cfg["current"]]["username"]
 job_name = cfg[cfg["current"]]["job_name"]
+ssh_private_key = cfg[cfg["current"]].get("ssh_private_key")
+ssh_user = cfg[cfg["current"]].get("ssh_user")
 
 server = jenkins.Jenkins(url, username=username)
 job = server.get_job_info(job_name, depth=0, fetch_all_builds=False)
@@ -71,9 +73,16 @@ def get_pr_status_label(task, short=True):
                         match = re.match(r"^(RHEL-\d+[.]\d+)[^/]+(/.+)$", label)
                         if match:
                             label = match.group(1) + match.group(2)
-                        match = re.match(r"^(CentOS-\d+)[^/]+(/.+)$", label)
-                        if match:
-                            label = match.group(1) + match.group(2)
+                        else:
+                            match = re.match(r"^(CentOS-Stream-\d+)(/.+)$", label)
+                            if match:
+                                label = match.group(1).replace(
+                                    "-Stream", ""
+                                ) + match.group(2)
+                            else:
+                                match = re.match(r"^(CentOS-\d+)[^/]+(/.+)$", label)
+                                if match:
+                                    label = match.group(1) + match.group(2)
                     return label
 
 
@@ -92,7 +101,7 @@ def get_pr_info(task):
 
 def get_queued_time(task):
     for action in task["actions"]:
-        if action["_class"] == "jenkins.metrics.impl.TimeInQueueAction":
+        if action.get("_class") == "jenkins.metrics.impl.TimeInQueueAction":
             return str(int(action["buildableDurationMillis"] / 1000))
 
 
@@ -221,9 +230,12 @@ def print_running_tasks(server, task_nums, args):
 def print_queued_tasks(server, task_nums, args):
     print(format_fields(None, "queued", None, True))
     queue_info = server.get_queue_info()
+    size = 0
     for task in queue_info:
+        size = size + 1
         if task["task"]["name"] == job_name:
             print(format_fields(task, "queued", None))
+    print(f"Queue size: {size}")
 
 
 def print_completed_tasks(server, task_nums, args):
@@ -251,6 +263,129 @@ def print_task_info(server, task_nums, args):
         yaml.safe_dump(task, sys.stdout)
 
 
+def get_node_info_for_task(server, task_num):
+    """Print the node info for the given task."""
+    task = server.get_build_info(job_name, task_num)
+    node_name = task["builtOn"]
+    node = server.get_node_info(node_name)
+    description = node["description"]
+    match = re.search(r"HOSTNAME=([0-9.]+)", description)
+    if match:
+        return {"node_name": node_name, "ip": match.group(1)}
+    else:
+        print(f"ERROR: for node_name {node_name} description unknown {description}")
+        return {"node_name": node_name, "ip": "unknown"}
+
+
+def get_task_tests_info(server, task_num):
+    """Get the info for all of the tests of a task given a task num."""
+    pat_pr = r"^ +lsr-github:github --pull-request= --pull-request=linux-system-roles:(?P<role>[^:]+):(?P<pr>[^:]+):.*$"
+    rx_pr = re.compile(pat_pr)
+    pat_test = (
+        r"^[|] dist-git-(?P<role>[^-]+)-[^/]+/tests/(?P<test>[^ ]+) +[|] +(?P<stage>[^ ]+) +[|]"
+        r" +(?P<state>[^ ]+) +[|] +(?P<result>[^ ]+) +[|] +(?P<arch>[^ ]+) +(?P<platform>[^ ]+) .*"
+    )
+    rx_test = re.compile(pat_test)
+    pat_guest_id = r" [|] +([a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}) +[|]"
+    rx_guest_id = re.compile(pat_guest_id)
+    pat_workspace = r"^Building remotely on .* in workspace ([^ ]+)$"
+    rx_workspace = re.compile(pat_workspace)
+    pat_work_test_dir = (
+        r"^.*dist-git-[^ ]+ working directory 'work-([^.]+.yml)([^']+)'.*$"
+    )
+    rx_work_test_dir = re.compile(pat_work_test_dir)
+    console = server.get_build_console_output(job_name, task_num)
+    console_lines = console.split("\n")
+    tests = {}
+    info = {"tests": tests}
+    pr = ""
+    role = ""
+    platform = ""
+    arch = ""
+    workspace = "unknown"
+    last_test = ""
+    for line in console_lines:
+        match = rx_guest_id.search(line)
+        if match and last_test:
+            tests[last_test]["guest_id"] = match.group(1)
+        elif last_test:
+            tests[last_test]["guest_id"] = "unknown"
+        match = rx_test.match(line)
+        if match:
+            test = match.group("test")
+            tests.setdefault(test, {}).update(match.groupdict())
+            arch = match.group("arch")
+            platform = match.group("platform")
+            last_test = test
+        else:
+            last_test = ""
+        match = rx_pr.match(line)
+        if match:
+            pr = match.group("pr")
+            role = match.group("role")
+        match = rx_workspace.match(line)
+        if match:
+            workspace = match.group(1)
+        match = rx_work_test_dir.match(line)
+        if match:
+            test = match.group(1)
+            work_dir = "work-" + test + match.group(2)
+            tests.setdefault(test, {})["work_dir"] = work_dir
+
+    info["pr"] = pr
+    info["role"] = role
+    info["platform"] = platform
+    info["arch"] = arch
+    info["workspace"] = workspace
+    node_info = get_node_info_for_task(server, task_num)
+    info.update(node_info)
+    return info
+
+
+def print_task_tests_info(server, task_nums, args):
+    """Print tests information for a given task."""
+    info = get_task_tests_info(server, int(args[0]))
+    print(
+        f"Role:{info['role']} PR:{info['pr']} Platform:{info['platform']} Arch:{info['arch']}"
+    )
+    print(f"Node:{info['node_name']} IP:{info['ip']} Workspace:{info['workspace']}")
+    fmt = "{:20s} {:10s} {:10s} {:36s} {} {}"
+    print(fmt.format("Stage", "State", "Result", "GuestID", "Test", "Workdir"))
+    for test, data in info["tests"].items():
+        print(
+            fmt.format(
+                data["stage"],
+                data["state"],
+                data["result"],
+                data["guest_id"],
+                test,
+                data.get("work_dir", "unknown"),
+            )
+        )
+
+
+def print_task_console(server, task_nums, args):
+    console = server.get_build_console_output(job_name, int(args[0]))
+    last = int(args[1])
+    console_lines = "\n".join(console.split("\n")[-last:])
+    print(console_lines)
+
+
+def print_node_info(server, task_nums, args):
+    """Print info for given build node."""
+    for node_name in args:
+        node = server.get_node_info(node_name)
+        yaml.safe_dump(node, sys.stdout)
+
+
+# def get_workspace_file(ip, workspace_dir, work_test_dir, guest_id, file_type):
+#     """Get the file from the workspace dir on the Jenkins server."""
+#     if file_type == "ansible":
+#         pass
+#     cmd = f"scp -i {ssh_private_key} {ssh_user}@{ip}:"
+#     subprocess.run(cmd)
+
+
 if len(sys.argv) > 1:
     locals()[sys.argv[1]](server, task_nums, sys.argv[2:])
 else: