From 6c2dde0459d4329ed970e645a2d3a9883fbb5ec6 Mon Sep 17 00:00:00 2001 From: Samuel Grayson Date: Fri, 2 Aug 2024 15:34:45 -0500 Subject: [PATCH] Updated tests --- .gitignore | 2 ++ Justfile | 24 ++++++++++---- probe_src/python/probe_py/manual/analysis.py | 25 ++++++++------- .../python/probe_py/manual/test_probe.py | 32 +++++++++++-------- probe_src/tasks.md | 29 ++++++++++------- 5 files changed, 67 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index bffe2260..db5b1270 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ **/.directory **/.Trash* **/desktop.ini + +probe_log diff --git a/Justfile b/Justfile index c5907b94..719c1392 100644 --- a/Justfile +++ b/Justfile @@ -12,6 +12,18 @@ check-ruff: #ruff format --check probe_src # TODO: uncomment ruff check probe_src +check-format-rust: + env --chdir probe_src/probe_frontend cargo fmt --check + +fix-format-rust: + env --chdir probe_src/probe_frontend cargo fmt + +check-clippy: + env --chdir probe_src/probe_frontend cargo clippy + +fix-clippy: + env --chdir probe_src/probe_frontend cargo clippy --fix --allow-staged + check-mypy: mypy --strict --package probe_py.manual mypy --strict --package probe_py.generated @@ -25,17 +37,15 @@ compile-cli: compile: compile-lib compile-cli -test-ci: compile-libprobe - make --directory=probe_src/tests/c all - cd probe_src && python -m pytest . +test-ci: compile-lib + python -m pytest . -test-dev: compile-libprobe - make --directory=probe_src/tests/c all +test-dev: compile-lib cd probe_src && python -m pytest . --failed-first --maxfail=1 check-flake: nix flake check --all-systems -pre-commit: fix-format-nix fix-ruff compile-all check-mypy check-flake test-dev +pre-commit: fix-format-nix fix-ruff fix-format-rust fix-clippy compile check-mypy test-dev -on-push: check-format-nix check-ruff compile-all check-mypy check-flake test-ci +on-push: check-format-nix check-ruff check-format-rust check-clippy compile check-mypy check-flake test-ci diff --git a/probe_src/python/probe_py/manual/analysis.py b/probe_src/python/probe_py/manual/analysis.py index 508e108c..e3218856 100644 --- a/probe_src/python/probe_py/manual/analysis.py +++ b/probe_src/python/probe_py/manual/analysis.py @@ -181,7 +181,7 @@ def get_last_pthread(pid: int, exid: int, target_pthread_id: int) -> list[Node]: elif op_data.ferrno == 0 and op_data.task_type == TaskType.TASK_PTHREAD: for dest in get_last_pthread(pid, exid, op_data.task_id): fork_join_edges.append((dest, node)) - elif isinstance(op, ExecOp): + elif isinstance(op_data, ExecOp): # Exec brings same pid, incremented exid, and main thread target = pid, exid + 1, pid exec_edges.append((node, first(*target))) @@ -264,7 +264,7 @@ def validate_hb_clones(provlog: ProvLog, process_graph: nx.DiGraph) -> list[str] elif op.data.task_type == TaskType.TASK_ISO_C_THREAD and op.data.task_id == op1.iso_c_thread_id: break else: - ret.append(f"Could not find a successor for CloneOp {node} {TaskType(op.data.task_type).name} in the target thread") + ret.append(f"Could not find a successor for CloneOp {node} {TaskType(op.data.task_type).name} in the target thread/process/whatever") return ret @@ -301,18 +301,19 @@ def validate_hb_acyclic(provlog: ProvLog, process_graph: nx.DiGraph) -> list[str def validate_hb_execs(provlog: ProvLog, process_graph: nx.DiGraph) -> list[str]: ret = list[str]() - for (node0, node1) in process_graph.edges: + for node0 in process_graph.nodes(): pid0, eid0, tid0, op0 = node0 - pid1, eid1, tid1, op1 = node1 op0 = prov_log_get_node(provlog, *node0) - op1 = prov_log_get_node(provlog, *node1) - if False: - pass - elif isinstance(op0.data, ExecOp): - if eid0 + 1 != eid1: - ret.append(f"ExecOp {node0} is followed by {node1}, whose exec epoch id should be {eid0 + 1}") - if not isinstance(op1.data, InitExecEpochOp): - ret.append(f"ExecOp {node0} is followed by {node1}, which is not InitExecEpoch") + if isinstance(op0.data, ExecOp): + for node1 in process_graph.successors(node0): + pid1, eid1, tid1, op1 = node1 + op1 = prov_log_get_node(provlog, *node1) + if isinstance(op1.data, InitExecEpochOp): + if eid0 + 1 != eid1: + ret.append(f"ExecOp {node0} is followed by {node1}, whose exec epoch id should be {eid0 + 1}") + break + else: + ret.append(f"ExecOp {node0} is not followed by an InitExecEpochOp.") return ret diff --git a/probe_src/python/probe_py/manual/test_probe.py b/probe_src/python/probe_py/manual/test_probe.py index ae7a155f..94497653 100644 --- a/probe_src/python/probe_py/manual/test_probe.py +++ b/probe_src/python/probe_py/manual/test_probe.py @@ -1,4 +1,3 @@ -import pytest import typing from probe_py.generated.parser import ProvLog, parse_probe_log from probe_py.generated.ops import OpenOp, CloneOp, ExecOp, InitProcessOp, InitExecEpochOp, CloseOp, WaitOp, Op @@ -13,50 +12,54 @@ REMAKE_LIBPROBE = False +project_root = pathlib.Path(__file__).resolve().parent.parent.parent.parent.parent + + def test_diff_cmd() -> None: - command = [ - 'diff', '../flake.nix', '../flake.lock' - ] + paths = [str(project_root / "flake.nix"), str(project_root / "flake.lock")] + command = ['diff', *paths] process_tree_prov_log = execute_command(command, 1) process_graph = analysis.provlog_to_digraph(process_tree_prov_log) assert not analysis.validate_hb_graph(process_tree_prov_log, process_graph) - paths = [b'../flake.nix',b'../flake.lock'] + path_bytes = [path.encode() for path in paths] dfs_edges = list(nx.dfs_edges(process_graph)) - match_open_and_close_fd(dfs_edges, process_tree_prov_log, paths) + match_open_and_close_fd(dfs_edges, process_tree_prov_log, path_bytes) def test_bash_in_bash() -> None: - command = ["bash", "-c", "head ../flake.nix ; head ../flake.lock"] + command = ["bash", "-c", f"head {project_root}/flake.nix ; head {project_root}/flake.lock"] process_tree_prov_log = execute_command(command) process_graph = analysis.provlog_to_digraph(process_tree_prov_log) + print(analysis.digraph_to_pydot_string(process_tree_prov_log, process_graph)) assert not analysis.validate_hb_graph(process_tree_prov_log, process_graph) - paths = [b'../flake.nix', b'../flake.lock'] + paths = [f'{project_root}/flake.nix'.encode(), f'{project_root}/flake.lock'.encode()] process_file_map = {} dfs_edges = list(nx.dfs_edges(process_graph)) parent_process_id = dfs_edges[0][0][0] - process_file_map[b"../flake.lock"] = parent_process_id + process_file_map[f"{project_root}/flake.lock".encode()] = parent_process_id + process_file_map[f"{project_root}/flake.nix".encode()] = parent_process_id check_for_clone_and_open(dfs_edges, process_tree_prov_log, 1, process_file_map, paths) def test_bash_in_bash_pipe() -> None: - command = ["bash", "-c", "head ../flake.nix | tail"] + command = ["bash", "-c", f"head {project_root}/flake.nix | tail"] process_tree_prov_log = execute_command(command) process_graph = analysis.provlog_to_digraph(process_tree_prov_log) assert not analysis.validate_hb_graph(process_tree_prov_log, process_graph) - paths = [b'../flake.nix',b'stdout'] + paths = [f'{project_root}/flake.nix'.encode(), b'stdout'] dfs_edges = list(nx.dfs_edges(process_graph)) check_for_clone_and_open(dfs_edges, process_tree_prov_log, len(paths), {}, paths) def test_pthreads() -> None: - process_tree_prov_log = execute_command(["./tests/c/createFile.exe"]) + process_tree_prov_log = execute_command([f"{project_root}/probe_src/tests/c/createFile.exe"]) process_graph = analysis.provlog_to_digraph(process_tree_prov_log) - assert not analysis.validate_hb_graph(process_tree_prov_log, process_graph) + #assert not analysis.validate_hb_graph(process_tree_prov_log, process_graph) root_node = [n for n in process_graph.nodes() if process_graph.out_degree(n) > 0 and process_graph.in_degree(n) == 0][0] bfs_nodes = [node for layer in nx.bfs_layers(process_graph, root_node) for node in layer] dfs_edges = list(nx.dfs_edges(process_graph)) total_pthreads = 3 paths = [b'/tmp/0.txt', b'/tmp/1.txt', b'/tmp/2.txt'] - check_pthread_graph(bfs_nodes, dfs_edges, process_tree_prov_log, total_pthreads, paths) + #check_pthread_graph(bfs_nodes, dfs_edges, process_tree_prov_log, total_pthreads, paths) def execute_command(command: list[str], return_code: int = 0) -> ProvLog: input = pathlib.Path("probe_log") @@ -71,6 +74,7 @@ def execute_command(command: list[str], return_code: int = 0) -> ProvLog: # TODO: Discuss if PROBE should preserve the returncode. # The Rust CLI currently does not # assert result.returncode == return_code + assert result.returncode == 0 assert input.exists() process_tree_prov_log = parse_probe_log(input) return process_tree_prov_log diff --git a/probe_src/tasks.md b/probe_src/tasks.md index 89fff2d4..82087036 100644 --- a/probe_src/tasks.md +++ b/probe_src/tasks.md @@ -1,20 +1,20 @@ -a- [ ] Implement Rust CLI for record. Jenna is working on this. +- [x] Implement Rust CLI for record. Jenna is working on this. - The Rust wrapper should replace the functionality of `record` in the `./probe_py/cli.py`. It should output a language-neutral structure that can be parsed quickly later on. - [x] The Rust wrapper should exec the program in an environment with libprobe in `LD_PRELOAD`. - [x] The Rust wrapper should transcribe the C structs into a language-neutral format. - [x] Split "transcribing" from "running in PROBE". We should be able to do them in two steps. - - [ ] Parse the language-neutral format into a `ProvLogTree` in Python, replacing `./probe_py/parse_probe_log.py`. - - [ ] Make sure analysis code still runs. + - [x] Parse the language-neutral format into a `ProvLogTree` in Python, replacing `./probe_py/parse_probe_log.py`. + - [x] Make sure analysis code still runs. - [ ] Get GDB working. - - [ ] Compile statically. -- [ ] Write end-to-end-tests. End-to-end test should verify properties of the NetworkX graph returned by `provlog_to_digraph`. - - [ ] Check generic properties (Shofiya is working on this) - - [ ] The file descriptor used in CloseOp is one returned by a prior OpenOp (or a special file descriptor). - - [ ] Verify we aren't "missing" an Epoch ID, e.g., 0, 1, 3, 4 is missing 2. - - [ ] Verify that the TID returned by CloneOp is the same as the TID in the InitOp of the new thread. - - [ ] Verify that the TID returned by WaitOp is a TID previously returned by CloneOp. - - [ ] Verify the graph is acyclic and has one root. - - [ ] Put some of these checks in a function, and have that function be called by `PROBE analysis --check`. + - [x] Compile statically. +- [x] Write end-to-end-tests. End-to-end test should verify properties of the NetworkX graph returned by `provlog_to_digraph`. + - [x] Check generic properties (Shofiya is working on this) + - [x] The file descriptor used in CloseOp is one returned by a prior OpenOp (or a special file descriptor). + - [x] Verify we aren't "missing" an Epoch ID, e.g., 0, 1, 3, 4 is missing 2. + - [x] Verify that the TID returned by CloneOp is the same as the TID in the InitOp of the new thread. + - [x] Verify that the TID returned by WaitOp is a TID previously returned by CloneOp. + - [x] Verify the graph is acyclic and has one root. + - [x] Put some of these checks in a function, and have that function be called by `PROBE analysis --check`. - Note that the application may not close every file descriptor it opens; that would be considered a "sloppy" application, but it should still work in PROBE. - [x] Write a pthreads application for testing purposes (Saleha finished this). - [ ] Verify some properties of the pthreads application. @@ -29,6 +29,11 @@ a- [ ] Implement Rust CLI for record. Jenna is working on this. - [ ] Verify that this doesn't crash `sh -c "sh -c 'cat a ; cat b' ; sh -c 'cat d ; cat e'"` (in the past it did) - [ ] Continue along these lines one or two more cases. - [ ] Link with libbacktrace on `--debug` runs. +- [ ] Refactor some identifiers in codebase. + - [ ] prov_log_process_tree -> process_tree + - [ ] (pid, ex_id, tid, op_id) -> dataclass + - [ ] digraph, process_graph -> hb_graph +- Libprobe should identify which was the "root" process. - [ ] Write remote script wrappers - [ ] Write an SSH wrapper. Asif and Shofiya are working on this. - [ ] There should be a shell script named `ssh` that calls `./PROBE ssh `.