From 6c74373db67160de3ca9add9f01beeec4afcc9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= Date: Mon, 2 Dec 2024 20:04:44 +0100 Subject: [PATCH 1/5] [analyzer] Include arch subdirectories in LD_LIBRARY_PATH for logging --- .../codechecker_analyzer/analyzer_context.py | 17 ++++++++++++----- .../tools/build-logger/tests/unit/__init__.py | 7 +++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index 16babf7b89..87648741df 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -71,13 +71,20 @@ def __init__(self): # Original caller environment of CodeChecker for external binaries self.__original_env = None - self.logger_lib_dir_path = os.path.join( - self._data_files_dir_path, 'ld_logger', 'lib') - - if not os.path.exists(self.logger_lib_dir_path): - self.logger_lib_dir_path = os.path.join( + # Find the path which has the architectures for the built ld_logger + # shared objects. + ld_logger_path = Path(self._data_files_dir_path, 'ld_logger', 'lib') + if not ld_logger_path.is_dir(): + ld_logger_path = Path( self._lib_dir_path, 'codechecker_analyzer', 'ld_logger', 'lib') + # Add the ld_logger_path and all children (architecture) paths to be + # later used in the LD_LIBRARY_PATH environment variable during + # logging of compiler invocations. + self.logger_lib_dir_path = ':'.join( + [str(ld_logger_path)] + + [str(arch) for arch in ld_logger_path.iterdir() if arch.is_dir()]) + self.logger_bin = None self.logger_file = None self.logger_compilers = None diff --git a/analyzer/tools/build-logger/tests/unit/__init__.py b/analyzer/tools/build-logger/tests/unit/__init__.py index e87a409252..73432fa04e 100644 --- a/analyzer/tools/build-logger/tests/unit/__init__.py +++ b/analyzer/tools/build-logger/tests/unit/__init__.py @@ -10,11 +10,11 @@ import json import os -import platform import shlex import subprocess import tempfile import unittest +from pathlib import Path from typing import Mapping, Optional, Tuple, Sequence REPO_ROOT = os.path.abspath(os.getenv("REPO_ROOT")) @@ -106,10 +106,13 @@ def read_actual_json(self) -> str: return fd.read() def get_envvars(self) -> Mapping[str, str]: + libdir = Path(LOGGER_DIR, "lib") return { "PATH": os.getenv("PATH"), "LD_PRELOAD": "ldlogger.so", - "LD_LIBRARY_PATH": os.path.join(LOGGER_DIR, "lib"), + "LD_LIBRARY_PATH": ':'.join([str(arch) for arch in + libdir.iterdir() + if arch.is_dir()]), "CC_LOGGER_GCC_LIKE": "gcc:g++:clang:clang++:/cc:c++", "CC_LOGGER_FILE": self.logger_file, "CC_LOGGER_DEBUG_FILE": self.logger_debug_file, From a53e57b7e4a5fe6ad05d7ecdcdbe28179466c3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= Date: Fri, 6 Dec 2024 14:42:48 +0100 Subject: [PATCH 2/5] Add mixed architecture test Fix and improve test case --- .../test_analyze_and_parse.py | 98 +++++++++++++++++++ .../test_files/mixed_arch_driver.c | 9 ++ .../analyze_and_parse/test_files/simple.c | 3 + 3 files changed, 110 insertions(+) create mode 100644 analyzer/tests/functional/analyze_and_parse/test_files/mixed_arch_driver.c create mode 100644 analyzer/tests/functional/analyze_and_parse/test_files/simple.c diff --git a/analyzer/tests/functional/analyze_and_parse/test_analyze_and_parse.py b/analyzer/tests/functional/analyze_and_parse/test_analyze_and_parse.py index 97a202eef7..6d24e4586e 100644 --- a/analyzer/tests/functional/analyze_and_parse/test_analyze_and_parse.py +++ b/analyzer/tests/functional/analyze_and_parse/test_analyze_and_parse.py @@ -39,6 +39,7 @@ def gen_test(path, mode): which compare the output of the command with the stored expected output. """ + def test(self): self.check_one_file(path, mode) return test @@ -787,3 +788,100 @@ def test_html_checker_url(self): encoding="utf-8", errors="ignore") as f: content = f.read() self.assertTrue(re.search('"url": ""', content)) + + def test_mixed_architecture_logging(self): + """ + Test if CodeChecker can properly log compilation commands when the + build process involves both 32-bit and 64-bit binaries acting as + build drivers. + + This verifies that the LD_LIBRARY_PATH setup in analyzer_context.py + correctly includes all architecture versions of the ld_logger.so + library, and that logging works with this setup. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + # We use a temporary directory, because we produce multiple files + # during this test, and it is easier to clean up. + mixed_arch_driver = os.path.join( + self.test_dir, + "mixed_arch_driver.c" + ) + simple_c = os.path.join( + self.test_dir, + "simple.c" + ) + + shutil.copy(mixed_arch_driver, tmp_dir) + shutil.copy(simple_c, tmp_dir) + + best_gcc_candidate_in_path = [ + path + for path in os.environ["PATH"].split(":") + if os.path.exists(os.path.join(path, "gcc")) + ] + if not best_gcc_candidate_in_path: + self.skipTest(f"No gcc candidate found in PATH:\ + {os.environ['PATH']}") + + try: + subprocess.check_call( + ["gcc", "-m32", "-c", "simple.c"], + cwd=tmp_dir, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as err: + self.skipTest(f"No 32-bit compilation support available:\ + {err.stderr}") + try: + subprocess.check_call( + ["gcc", "-m64", "-c", "simple.c"], + cwd=tmp_dir, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as err: + self.skipTest(f"No 64-bit compilation support available:\ + {err.stderr}") + + subprocess.check_call( + ["gcc", "-m32", "mixed_arch_driver.c", "-o", "driver32"], + cwd=tmp_dir + ) + subprocess.check_call( + ["gcc", "-m64", "mixed_arch_driver.c", "-o", "driver64"], + cwd=tmp_dir + ) + + log_file = os.path.join(tmp_dir, "compile_commands.json") + cmd = [ + "CodeChecker", "log", "-b", "./driver32;./driver64", + "-o", log_file, + ] + + _, err, returncode = call_command(cmd, cwd=tmp_dir, + env=self.env) + + self.assertEqual(returncode, 0, f"CodeChecker log failed:\ + {err}") + + # Verify the logged commands + with open(log_file, "r", encoding="utf-8") as f: + logged_commands = json.load(f) + + # The buildlog should have 4 commands - 2 from each driver + # (and for each driver there is one with a '-m32' and one with a + # '-m64' flag) + self.assertEqual( + len(logged_commands), 4, f"Logged commands: {logged_commands}" + ) + + commands = [entry["command"] for entry in logged_commands] + self.assertTrue( + 2 == len([cmd for cmd in commands if "-m32" in cmd]), + f"Expected 2 32-bit compilations. Logged commands:\ + {logged_commands}" + ) + self.assertTrue( + 2 == len([cmd for cmd in commands if "-m64" in cmd]), + f"Expected 2 64-bit compilations. Logged commands:\ + {logged_commands}" + ) diff --git a/analyzer/tests/functional/analyze_and_parse/test_files/mixed_arch_driver.c b/analyzer/tests/functional/analyze_and_parse/test_files/mixed_arch_driver.c new file mode 100644 index 0000000000..090291cb6b --- /dev/null +++ b/analyzer/tests/functional/analyze_and_parse/test_files/mixed_arch_driver.c @@ -0,0 +1,9 @@ +#include + +int main(void) { + // These commands are intended to test CodeChecker's ability to build-log + // cross compilations. + system("gcc -m32 simple.c -o simple32"); + system("gcc -m64 simple.c -o simple64"); + return 0; +} \ No newline at end of file diff --git a/analyzer/tests/functional/analyze_and_parse/test_files/simple.c b/analyzer/tests/functional/analyze_and_parse/test_files/simple.c new file mode 100644 index 0000000000..f02338e75f --- /dev/null +++ b/analyzer/tests/functional/analyze_and_parse/test_files/simple.c @@ -0,0 +1,3 @@ +int main(void) { + return 0; +} \ No newline at end of file From 8ef3a6f3f86d1ae593921e4405c81bacaaf411c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= Date: Fri, 6 Dec 2024 14:46:47 +0100 Subject: [PATCH 3/5] Only add children paths to lib_dir_path adsf --- .../codechecker_analyzer/analyzer_context.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index 87648741df..2cacdd9ef4 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -73,17 +73,18 @@ def __init__(self): # Find the path which has the architectures for the built ld_logger # shared objects. - ld_logger_path = Path(self._data_files_dir_path, 'ld_logger', 'lib') + ld_logger_path = Path(self._data_files_dir_path, "ld_logger", "lib") if not ld_logger_path.is_dir(): ld_logger_path = Path( - self._lib_dir_path, 'codechecker_analyzer', 'ld_logger', 'lib') - - # Add the ld_logger_path and all children (architecture) paths to be - # later used in the LD_LIBRARY_PATH environment variable during - # logging of compiler invocations. - self.logger_lib_dir_path = ':'.join( - [str(ld_logger_path)] + - [str(arch) for arch in ld_logger_path.iterdir() if arch.is_dir()]) + self._lib_dir_path, "codechecker_analyzer", "ld_logger", "lib" + ) + + # Add all children (architecture) paths to be later used in the + # LD_LIBRARY_PATH environment variable during logging of compiler + # invocations. + self.logger_lib_dir_path = ":".join( + [str(arch) for arch in ld_logger_path.iterdir() if arch.is_dir()] + ) self.logger_bin = None self.logger_file = None From a5e14b76431a9484b97530d2f0592e698aec457d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= Date: Fri, 6 Dec 2024 15:18:06 +0100 Subject: [PATCH 4/5] Simplify ld_logger intall folder structure fix directory paths --- analyzer/tools/build-logger/Makefile | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/analyzer/tools/build-logger/Makefile b/analyzer/tools/build-logger/Makefile index 2174201837..e9ed753148 100644 --- a/analyzer/tools/build-logger/Makefile +++ b/analyzer/tools/build-logger/Makefile @@ -76,24 +76,18 @@ LIB_DIR = $(BUILD_DIR)/lib all: ldlogger ldlogger_32.so ldlogger_64.so pack32bit pack64bit pack32bit: 32bit packbin - for x86dir in 'i386' 'i486' 'i586' 'i686'; do \ - mkdir -p $(LIB_DIR)/$$x86dir ; \ - cp ldlogger_32.so $(LIB_DIR)/$$x86dir/ldlogger.so ; \ - done + mkdir -p $(LIB_DIR)/32bit ; \ + cp ldlogger_32.so $(LIB_DIR)/32bit/ldlogger.so ; \ rm -f ldlogger_32.so pack64bit: 64bit packbin - for x8664dir in 'x86_64'; do \ - mkdir -p $(LIB_DIR)/$$x8664dir ; \ - cp ldlogger_64.so $(LIB_DIR)/$$x8664dir/ldlogger.so ; \ - done + mkdir -p $(LIB_DIR)/64bit ; \ + cp ldlogger_64.so $(LIB_DIR)/64bit/ldlogger.so ; \ rm -f ldlogger_64.so pack64bit_only: 64bit_only packbin64 - for x8664dir in 'x86_64'; do \ - mkdir -p $(LIB_DIR)/$$x8664dir ; \ - cp ldlogger_64.so $(LIB_DIR)/$$x8664dir/ldlogger.so ; \ - done + mkdir -p $(LIB_DIR)/64bit ; \ + cp ldlogger_64.so $(LIB_DIR)/64bit/ldlogger.so ; \ rm -f ldlogger_64.so # pack binary From 4817933d55be744da8854f4bd4b7f7f5f8c49333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= Date: Fri, 6 Dec 2024 15:27:05 +0100 Subject: [PATCH 5/5] Update docs to reflect ld_logger options --- docs/analyzer/user_guide.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/analyzer/user_guide.md b/docs/analyzer/user_guide.md index 55b07a4ed7..00b58dcfef 100644 --- a/docs/analyzer/user_guide.md +++ b/docs/analyzer/user_guide.md @@ -793,7 +793,10 @@ If this is not possible, you can work around the situation by specifying the absolute path of the `ldlogger.so` in the `LD_PRELOAD`: ```sh -LD_PRELOAD=/ld_logger/lib/x86_64/ldlogger.so CodeChecker log -o compile_commands.json -b "make -j2" +# For 64-bit compilers +LD_PRELOAD=/ld_logger/lib/64bit/ldlogger.so CodeChecker log -o compile_commands.json -b "make -j2" +# For 32-bit compilers +LD_PRELOAD=/ld_logger/lib/32bit/ldlogger.so CodeChecker log -o compile_commands.json -b "make -j2" ``` #### Change user inside the build command @@ -2718,4 +2721,4 @@ The following actions are available: setting. If none of the filter options is provided, then that setting is not applied on -any report. +any report. \ No newline at end of file