From 28bdac683952da46307c6f970b4fd00475b7609a Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Wed, 9 Oct 2024 15:32:41 -0700 Subject: [PATCH 1/8] support comma separate list when using --exclude --- src/auditwheel/main_repair.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index 4374f208..9bcd0388 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -109,7 +109,11 @@ def execute(args, p): from .repair import repair_wheel from .wheel_abi import NonPlatformWheel, analyze_wheel_abi - exclude = frozenset(args.EXCLUDE) + # Split comma separated values and flatten it + exclude = frozenset( + item.strip() for sublist in args.EXCLUDE for item in sublist.split(',') + ) + wheel_policy = WheelPolicies() for wheel_file in args.WHEEL_FILE: From f66f25773609da7edd5e5ffae9e7e883f2b1d7ee Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Wed, 9 Oct 2024 15:32:59 -0700 Subject: [PATCH 2/8] update test case --- tests/integration/testrpath/c/c.c | 3 +++ tests/integration/testrpath/c/c.h | 1 + tests/integration/testrpath/setup.py | 14 +++++++++----- .../testrpath/src/testrpath/testrpath.c | 3 ++- 4 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 tests/integration/testrpath/c/c.c create mode 100644 tests/integration/testrpath/c/c.h diff --git a/tests/integration/testrpath/c/c.c b/tests/integration/testrpath/c/c.c new file mode 100644 index 00000000..442b08d5 --- /dev/null +++ b/tests/integration/testrpath/c/c.c @@ -0,0 +1,3 @@ +int fc(void) { + return 20; +} diff --git a/tests/integration/testrpath/c/c.h b/tests/integration/testrpath/c/c.h new file mode 100644 index 00000000..ce519fda --- /dev/null +++ b/tests/integration/testrpath/c/c.h @@ -0,0 +1 @@ +int fc(void); \ No newline at end of file diff --git a/tests/integration/testrpath/setup.py b/tests/integration/testrpath/setup.py index 75b92674..2e3b1a5e 100644 --- a/tests/integration/testrpath/setup.py +++ b/tests/integration/testrpath/setup.py @@ -11,10 +11,14 @@ class BuildExt(build_ext): def run(self) -> None: cmd = "gcc -fPIC -shared -o b/libb.so b/b.c" subprocess.check_call(cmd.split()) + + cmd = "gcc -fPIC -shared -o c/libc.so c/c.c" + subprocess.check_call(cmd.split()) + cmd = ( "gcc -fPIC -shared -o a/liba.so " - "-Wl,{dtags_flag} -Wl,-rpath=$ORIGIN/../b " - "-Ib a/a.c -Lb -lb" + "-Wl,{dtags_flag} -Wl,-rpath=$ORIGIN/../b -Wl,-rpath=$ORIGIN/../c " + "-Ib a/a.c -Lb -Lc -lb -lc" ).format( dtags_flag=( "--enable-new-dtags" @@ -36,9 +40,9 @@ def run(self) -> None: Extension( "testrpath/testrpath", sources=["src/testrpath/testrpath.c"], - include_dirs=["a"], - libraries=["a"], - library_dirs=["a"], + include_dirs=["a", "c"], + libraries=["a", "c"], + library_dirs=["a", "c"], ) ], ) diff --git a/tests/integration/testrpath/src/testrpath/testrpath.c b/tests/integration/testrpath/src/testrpath/testrpath.c index 32731129..c4d2cb50 100644 --- a/tests/integration/testrpath/src/testrpath/testrpath.c +++ b/tests/integration/testrpath/src/testrpath/testrpath.c @@ -1,5 +1,6 @@ #include #include "a.h" +#include "c.h" static PyObject * func(PyObject *self, PyObject *args) @@ -9,7 +10,7 @@ func(PyObject *self, PyObject *args) (void)self; (void)args; - res = fa(); + res = fa() + fc(); return PyLong_FromLong(res); } From cc79c8ac8fd46e5425d4ecf2c06119a82df422a7 Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Wed, 9 Oct 2024 16:50:47 -0700 Subject: [PATCH 3/8] update test case to test comma sepparated list --- tests/integration/test_manylinux.py | 59 ++++++++++++------- tests/integration/testrpath/c/c.h | 1 - tests/integration/testrpath/setup.py | 16 +++-- .../testrpath/src/testrpath/testrpath.c | 4 +- tests/integration/testrpath/{c/c.c => x/x.c} | 2 +- tests/integration/testrpath/x/x.h | 1 + 6 files changed, 52 insertions(+), 31 deletions(-) delete mode 100644 tests/integration/testrpath/c/c.h rename tests/integration/testrpath/{c/c.c => x/x.c} (53%) create mode 100644 tests/integration/testrpath/x/x.h diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index acd7b1ab..860c7474 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -329,27 +329,44 @@ def test_repair_exclude(self, any_manylinux_container, io_folder): orig_wheel = filenames[0] assert "manylinux" not in orig_wheel - repair_command = [ - f"LD_LIBRARY_PATH={test_path}/a:$LD_LIBRARY_PATH", - "auditwheel", - "repair", - f"--plat={policy}", - "--only-plat", - "-w", - "/io", - "--exclude=liba.so", - f"/io/{orig_wheel}", - ] - output = docker_exec(manylinux_ctr, ["bash", "-c", " ".join(repair_command)]) - assert "Excluding liba.so" in output - filenames = os.listdir(io_folder) - assert len(filenames) == 2 - repaired_wheel = f"testrpath-0.0.1-{PYTHON_ABI}-{tag}.whl" - assert repaired_wheel in filenames - - # Make sure we don't have liba.so & libb.so in the result - contents = zipfile.ZipFile(os.path.join(io_folder, repaired_wheel)).namelist() - assert not any(x for x in contents if "/liba" in x or "/libb" in x) + def run_repair_test(exclude_args, expected_exclusions): + repair_command = [ + f"LD_LIBRARY_PATH={test_path}/a:$LD_LIBRARY_PATH", + "auditwheel", + "repair", + f"--plat={policy}", + "--only-plat", + "-w", + "/io", + ] + exclude_args + [ + f"/io/{orig_wheel}", + ] + output = docker_exec(manylinux_ctr, ["bash", "-c", " ".join(repair_command)]) + + # Check for exclusions in the output + for arg in exclude_args: + if ',' in arg: + libs = arg.split('=')[1].split(',') + for lib in libs: + assert f"Excluding {lib}" in output + else: + lib = arg.split('=')[1] + assert f"Excluding {lib}" in output + + filenames = os.listdir(io_folder) + assert len(filenames) == 2 + repaired_wheel = f"testrpath-0.0.1-{PYTHON_ABI}-{tag}.whl" + assert repaired_wheel in filenames + + # Make sure we don't have the excluded libraries in the result + contents = zipfile.ZipFile(os.path.join(io_folder, repaired_wheel)).namelist() + for lib in expected_exclusions: + assert not any(x for x in contents if f"/{lib}" in x) + + # Test case 1: Exclude liba.so only - it will exclude libb.so as well due to indirect reference + run_repair_test(["--exclude=liba.so", "--exclude=libx.so"], ["liba.so", "libb.so", "libx.so"]) + # Test case 2: Exclude liba.so and libx.so using comma separated + run_repair_test(["--exclude=liba.so,libx.so"], ["liba.so", "libb.so", "libx.so"]) def test_build_wheel_with_binary_executable( self, any_manylinux_container, docker_python, io_folder diff --git a/tests/integration/testrpath/c/c.h b/tests/integration/testrpath/c/c.h deleted file mode 100644 index ce519fda..00000000 --- a/tests/integration/testrpath/c/c.h +++ /dev/null @@ -1 +0,0 @@ -int fc(void); \ No newline at end of file diff --git a/tests/integration/testrpath/setup.py b/tests/integration/testrpath/setup.py index 2e3b1a5e..fc043cd7 100644 --- a/tests/integration/testrpath/setup.py +++ b/tests/integration/testrpath/setup.py @@ -12,13 +12,13 @@ def run(self) -> None: cmd = "gcc -fPIC -shared -o b/libb.so b/b.c" subprocess.check_call(cmd.split()) - cmd = "gcc -fPIC -shared -o c/libc.so c/c.c" + cmd = "gcc -fPIC -shared -o x/libx.so x/x.c" subprocess.check_call(cmd.split()) cmd = ( "gcc -fPIC -shared -o a/liba.so " - "-Wl,{dtags_flag} -Wl,-rpath=$ORIGIN/../b -Wl,-rpath=$ORIGIN/../c " - "-Ib a/a.c -Lb -Lc -lb -lc" + "-Wl,{dtags_flag} -Wl,-rpath=$ORIGIN/../b " + "-Ib a/a.c -Lb -lb" ).format( dtags_flag=( "--enable-new-dtags" @@ -40,9 +40,13 @@ def run(self) -> None: Extension( "testrpath/testrpath", sources=["src/testrpath/testrpath.c"], - include_dirs=["a", "c"], - libraries=["a", "c"], - library_dirs=["a", "c"], + include_dirs=["a", "x"], + libraries=["a", "x"], + library_dirs=["a", "x"], + extra_link_args=[ + '-Wl,-rpath,$ORIGIN/../a', + '-Wl,-rpath,$ORIGIN/../x' + ], ) ], ) diff --git a/tests/integration/testrpath/src/testrpath/testrpath.c b/tests/integration/testrpath/src/testrpath/testrpath.c index c4d2cb50..e406a92e 100644 --- a/tests/integration/testrpath/src/testrpath/testrpath.c +++ b/tests/integration/testrpath/src/testrpath/testrpath.c @@ -1,6 +1,6 @@ #include #include "a.h" -#include "c.h" +#include "x.h" static PyObject * func(PyObject *self, PyObject *args) @@ -10,7 +10,7 @@ func(PyObject *self, PyObject *args) (void)self; (void)args; - res = fa() + fc(); + res = fa() + fx(); return PyLong_FromLong(res); } diff --git a/tests/integration/testrpath/c/c.c b/tests/integration/testrpath/x/x.c similarity index 53% rename from tests/integration/testrpath/c/c.c rename to tests/integration/testrpath/x/x.c index 442b08d5..2f6eaed7 100644 --- a/tests/integration/testrpath/c/c.c +++ b/tests/integration/testrpath/x/x.c @@ -1,3 +1,3 @@ -int fc(void) { +int fx(void) { return 20; } diff --git a/tests/integration/testrpath/x/x.h b/tests/integration/testrpath/x/x.h new file mode 100644 index 00000000..1c443e7a --- /dev/null +++ b/tests/integration/testrpath/x/x.h @@ -0,0 +1 @@ +int fx(void); From 9c208055ef001be924667be456afc522a4fd3a2f Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Wed, 9 Oct 2024 17:02:01 -0700 Subject: [PATCH 4/8] lint and fix comment --- src/auditwheel/main_repair.py | 2 +- tests/integration/test_manylinux.py | 51 +++++++++++++++++----------- tests/integration/testrpath/setup.py | 5 +-- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index 9bcd0388..07faca87 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -111,7 +111,7 @@ def execute(args, p): # Split comma separated values and flatten it exclude = frozenset( - item.strip() for sublist in args.EXCLUDE for item in sublist.split(',') + item.strip() for sublist in args.EXCLUDE for item in sublist.split(",") ) wheel_policy = WheelPolicies() diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 860c7474..55a670e9 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -330,27 +330,33 @@ def test_repair_exclude(self, any_manylinux_container, io_folder): assert "manylinux" not in orig_wheel def run_repair_test(exclude_args, expected_exclusions): - repair_command = [ - f"LD_LIBRARY_PATH={test_path}/a:$LD_LIBRARY_PATH", - "auditwheel", - "repair", - f"--plat={policy}", - "--only-plat", - "-w", - "/io", - ] + exclude_args + [ - f"/io/{orig_wheel}", - ] - output = docker_exec(manylinux_ctr, ["bash", "-c", " ".join(repair_command)]) + repair_command = ( + [ + f"LD_LIBRARY_PATH={test_path}/a:$LD_LIBRARY_PATH", + "auditwheel", + "repair", + f"--plat={policy}", + "--only-plat", + "-w", + "/io", + ] + + exclude_args + + [ + f"/io/{orig_wheel}", + ] + ) + output = docker_exec( + manylinux_ctr, ["bash", "-c", " ".join(repair_command)] + ) # Check for exclusions in the output for arg in exclude_args: - if ',' in arg: - libs = arg.split('=')[1].split(',') + if "," in arg: + libs = arg.split("=")[1].split(",") for lib in libs: assert f"Excluding {lib}" in output else: - lib = arg.split('=')[1] + lib = arg.split("=")[1] assert f"Excluding {lib}" in output filenames = os.listdir(io_folder) @@ -359,14 +365,21 @@ def run_repair_test(exclude_args, expected_exclusions): assert repaired_wheel in filenames # Make sure we don't have the excluded libraries in the result - contents = zipfile.ZipFile(os.path.join(io_folder, repaired_wheel)).namelist() + contents = zipfile.ZipFile( + os.path.join(io_folder, repaired_wheel) + ).namelist() for lib in expected_exclusions: assert not any(x for x in contents if f"/{lib}" in x) - # Test case 1: Exclude liba.so only - it will exclude libb.so as well due to indirect reference - run_repair_test(["--exclude=liba.so", "--exclude=libx.so"], ["liba.so", "libb.so", "libx.so"]) + # Test case 1: Exclude liba.so and libx.so using 2 --exclude parameters + run_repair_test( + ["--exclude=liba.so", "--exclude=libx.so"], + ["liba.so", "libb.so", "libx.so"], + ) # Test case 2: Exclude liba.so and libx.so using comma separated - run_repair_test(["--exclude=liba.so,libx.so"], ["liba.so", "libb.so", "libx.so"]) + run_repair_test( + ["--exclude=liba.so,libx.so"], ["liba.so", "libb.so", "libx.so"] + ) def test_build_wheel_with_binary_executable( self, any_manylinux_container, docker_python, io_folder diff --git a/tests/integration/testrpath/setup.py b/tests/integration/testrpath/setup.py index fc043cd7..7264fc6e 100644 --- a/tests/integration/testrpath/setup.py +++ b/tests/integration/testrpath/setup.py @@ -43,10 +43,7 @@ def run(self) -> None: include_dirs=["a", "x"], libraries=["a", "x"], library_dirs=["a", "x"], - extra_link_args=[ - '-Wl,-rpath,$ORIGIN/../a', - '-Wl,-rpath,$ORIGIN/../x' - ], + extra_link_args=["-Wl,-rpath,$ORIGIN/../a", "-Wl,-rpath,$ORIGIN/../x"], ) ], ) From 8c9f44e01edde33953ef0e47a03fd3f357194c93 Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Thu, 10 Oct 2024 01:44:51 -0700 Subject: [PATCH 5/8] fix tests --- tests/integration/test_manylinux.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 55a670e9..4e7ad201 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -522,8 +522,12 @@ def test_build_wheel_depending_on_library_with_rpath( [ "bash", "-c", - "LD_LIBRARY_PATH=" - "/auditwheel_src/tests/integration/testrpath/a:$LD_LIBRARY_PATH " + ( + "LD_LIBRARY_PATH=" + "/auditwheel_src/tests/integration/testrpath/a:" + "/auditwheel_src/tests/integration/testrpath/x:" + "$LD_LIBRARY_PATH " + ) + repair_command, ], ) @@ -542,13 +546,15 @@ def test_build_wheel_depending_on_library_with_rpath( "from testrpath import testrpath; print(testrpath.func())", ], ) - assert output.strip() == "11" + # output = fa + fx = (1+10) + 20 = 31 + assert output.strip() == "31" with zipfile.ZipFile(os.path.join(io_folder, repaired_wheel)) as w: libraries = tuple( name for name in w.namelist() if "testrpath.libs/lib" in name ) - assert len(libraries) == 2 + assert len(libraries) == 3 assert any(".libs/liba" in name for name in libraries) + assert any(".libs/libx" in name for name in libraries) for name in libraries: with w.open(name) as f: elf = ELFFile(io.BytesIO(f.read())) From fdfd7d009a3d40f17045f40cb40e5cc556941ed5 Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Thu, 10 Oct 2024 08:03:05 -0700 Subject: [PATCH 6/8] fix test_build_wheel_with_binary_executable to calculate the right expected python path name --- tests/integration/test_manylinux.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 4e7ad201..58a8c46a 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -440,9 +440,17 @@ def test_build_wheel_with_binary_executable( ) # testprogram should be a Python shim since we had to rewrite its RPATH. + python_version = docker_exec( + docker_python, + [ + "python", + "-c", + "import sys; print(f'python{sys.version_info.major}.{sys.version_info.minor}')", + ], + ).strip() assert ( docker_exec(docker_python, ["head", "-n1", "/usr/local/bin/testprogram"]) - == "#!/usr/local/bin/python\n" + == f"#!/usr/local/bin/{python_version}\n" ) # testprogram_nodeps should be the unmodified ELF binary. From 9112445fefe48a195331a3205e9d4e45b9c4a3fb Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Thu, 10 Oct 2024 08:11:00 -0700 Subject: [PATCH 7/8] lint --- tests/integration/test_manylinux.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 58a8c46a..80fb0b81 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -445,7 +445,11 @@ def test_build_wheel_with_binary_executable( [ "python", "-c", - "import sys; print(f'python{sys.version_info.major}.{sys.version_info.minor}')", + ( + "import sys; " + "print(f'python{sys.version_info.major}." + "{sys.version_info.minor}')" + ), ], ).strip() assert ( From ce36fd071952cc1f5f08ed10a29b17feea319c01 Mon Sep 17 00:00:00 2001 From: Peter Pham Date: Thu, 10 Oct 2024 10:20:29 -0700 Subject: [PATCH 8/8] Trigger rebuild