Fix: Patch RPATHs of non-Python extension dependencies

auditwheel/wheel_abi.py

@@ -118,10 +118,16 @@ def get_wheel_elfdata(wheel_fn: str):
             # we should walk its elftree.
             if basename(fn) not in needed_libs:
                 full_elftree[fn] = nonpy_elftree[fn]
-                full_external_refs[fn] = lddtree_external_references(
-                    nonpy_elftree[fn], ctx.path)
 
-    log.debug(json.dumps(full_elftree, indent=4))
+            # Even if a non-pyextension ELF file is not needed, we
+            # should include it as an external references, because
+            # they might also require external libraries.
+            full_external_refs[fn] = lddtree_external_references(
+                nonpy_elftree[fn], ctx.path)
+
+    log.debug('full_elftree:\n%s', json.dumps(full_elftree, indent=4))
+    log.debug('full_external_refs (will be repaired):\n%s',
+              json.dumps(full_external_refs, indent=4))
 
     return (full_elftree, full_external_refs, versioned_symbols,
             uses_ucs2_symbols, uses_PyFPE_jbuf)

tests/integration/nonpy_rpath/README.md

@@ -0,0 +1,9 @@
+# Python 3 extension with non-Python library dependency
+
+This example was inspired from https://gist.github.com/physacco/2e1b52415f3a964ad2a542a99bebed8f
+
+This test extension builds two libraries: `_hello.*.so` and `lib_zlibexample.*.so`, where the `*` is a string composed of Python ABI versions and platform tags.
+
+The extension `lib_zlibexample.*.so` should be repaired by auditwheel because it is a needed library, even though it is not a Python extension.
+
+[Issue #136](https://github.com/pypa/auditwheel/issues/136) documents the underlying problem that this test case is designed to solve.

tests/integration/nonpy_rpath/extensions/.gitignore

@@ -0,0 +1 @@
+testzlib

tests/integration/nonpy_rpath/extensions/testzlib.cpp

@@ -0,0 +1,137 @@
+// Copyright 2007 Timo Bingmann <[email protected]>
+// Distributed under the Boost Software License, Version 1.0.
+// (See http://www.boost.org/LICENSE_1_0.txt)
+// Taken from https://panthema.net/2007/0328-ZLibString.html
+
+#include <string.h>
+#include <stdexcept>
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+
+#include <zlib.h>
+#include "testzlib.h"
+
+/** Compress a STL string using zlib with given compression level and return
+  * the binary data. */
+std::string compress_string(const std::string& str,
+                            int compressionlevel)
+{
+    z_stream zs;                        // z_stream is zlib's control structure
+    memset(&zs, 0, sizeof(zs));
+
+    if (deflateInit(&zs, compressionlevel) != Z_OK)
+        throw(std::runtime_error("deflateInit failed while compressing."));
+
+    zs.next_in = (Bytef*)str.data();
+    zs.avail_in = str.size();           // set the z_stream's input
+
+    int ret;
+    char outbuffer[32768];
+    std::string outstring;
+
+    // retrieve the compressed bytes blockwise
+    do {
+        zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
+        zs.avail_out = sizeof(outbuffer);
+
+        ret = deflate(&zs, Z_FINISH);
+
+        if (outstring.size() < zs.total_out) {
+            // append the block to the output string
+            outstring.append(outbuffer,
+                             zs.total_out - outstring.size());
+        }
+    } while (ret == Z_OK);
+
+    deflateEnd(&zs);
+
+    if (ret != Z_STREAM_END) {          // an error occurred that was not EOF
+        std::ostringstream oss;
+        oss << "Exception during zlib compression: (" << ret << ") " << zs.msg;
+        throw(std::runtime_error(oss.str()));
+    }
+
+    return outstring;
+}
+
+/** Decompress an STL string using zlib and return the original data. */
+std::string decompress_string(const std::string& str)
+{
+    z_stream zs;                        // z_stream is zlib's control structure
+    memset(&zs, 0, sizeof(zs));
+
+    if (inflateInit(&zs) != Z_OK)
+        throw(std::runtime_error("inflateInit failed while decompressing."));
+
+    zs.next_in = (Bytef*)str.data();
+    zs.avail_in = str.size();
+
+    int ret;
+    char outbuffer[32768];
+    std::string outstring;
+
+    // get the decompressed bytes blockwise using repeated calls to inflate
+    do {
+        zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
+        zs.avail_out = sizeof(outbuffer);
+
+        ret = inflate(&zs, 0);
+
+        if (outstring.size() < zs.total_out) {
+            outstring.append(outbuffer,
+                             zs.total_out - outstring.size());
+        }
+
+    } while (ret == Z_OK);
+
+    inflateEnd(&zs);
+
+    if (ret != Z_STREAM_END) {          // an error occurred that was not EOF
+        std::ostringstream oss;
+        oss << "Exception during zlib decompression: (" << ret << ") "
+            << zs.msg;
+        throw(std::runtime_error(oss.str()));
+    }
+
+    return outstring;
+}
+
+/** Small dumb tool (de)compressing cin to cout. It holds all input in memory,
+  * so don't use it for huge files. */
+int main(int argc, char* argv[])
+{
+    std::string allinput;
+
+    while (std::cin.good())     // read all input from cin
+    {
+        char inbuffer[32768];
+        std::cin.read(inbuffer, sizeof(inbuffer));
+        allinput.append(inbuffer, std::cin.gcount());
+    }
+
+    if (argc >= 2 && strcmp(argv[1], "-d") == 0)
+    {
+        std::string cstr = decompress_string( allinput );
+
+        std::cerr << "Inflated data: "
+                  << allinput.size() << " -> " << cstr.size()
+                  << " (" << std::setprecision(1) << std::fixed
+                  << ( ((float)cstr.size() / (float)allinput.size() - 1.0) * 100.0 )
+                  << "% increase).\n";
+
+        std::cout << cstr;
+    }
+    else
+    {
+        std::string cstr = compress_string( allinput );
+
+        std::cerr << "Deflated data: "
+                  << allinput.size() << " -> " << cstr.size()
+                  << " (" << std::setprecision(1) << std::fixed
+                  << ( (1.0 - (float)cstr.size() / (float)allinput.size()) * 100.0)
+                  << "% saved).\n";
+
+        std::cout << cstr;
+    }
+}

tests/integration/nonpy_rpath/extensions/testzlib.h

@@ -0,0 +1,11 @@
+#include <iostream>
+#include <zlib.h>
+
+#ifndef ZLIB_EXAMPLE // include guard
+#define ZLIB_EXAMPLE
+
+std::string compress_string(const std::string& str,
+                            int compressionlevel = Z_BEST_COMPRESSION);
+std::string decompress_string(const std::string& str);
+
+#endif /* ZLIB_EXAMPLE */

tests/integration/nonpy_rpath/extensions/testzlib.sh

@@ -0,0 +1,8 @@
+# compile and run
+g++ testzlib.cpp -lz -o testzlib
+if [ $? == 0 ]; then
+    echo Hello Hello Hello Hello Hello Hello! | ./testzlib | ./testzlib -d
+fi
+# Deflated data: 37 -> 19 (48.6% saved).
+# Inflated data: 19 -> 37 (94.7% increase).
+# Hello Hello Hello Hello Hello Hello!

tests/integration/nonpy_rpath/hello.cpp

@@ -0,0 +1,97 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "extensions/testzlib.h"
+
+// Module method definitions
+static PyObject* hello_world(PyObject *self, PyObject *args) {
+    printf("Hello, World!");
+    Py_RETURN_NONE;
+}
+
+// static PyObject* zlib_example(PyObject *self, PyObject *args) {
+//     main();
+//     Py_RETURN_NONE;
+// }
+
+static PyObject* z_compress(PyObject *self, PyObject *args) {
+    const char* str_compress;
+    if (!PyArg_ParseTuple(args, "s", &str_compress)) {
+        return NULL;
+    }
+
+    std::string str_compress_s = str_compress;
+    std::string compressed = compress_string(str_compress_s);
+    // Copy pointer (compressed string may contain 0 byte)
+    const char * str_compressed = &*compressed.begin();
+    return PyBytes_FromStringAndSize(str_compressed, compressed.length());
+}
+
+static PyObject* z_uncompress(PyObject *self, PyObject *args) {
+    const char * str_uncompress;
+    Py_ssize_t str_uncompress_len;
+    // according to https://docs.python.org/3/c-api/arg.html
+    if (!PyArg_ParseTuple(args, "y#", &str_uncompress, &str_uncompress_len)) {
+        return NULL;
+    }
+
+    std::string uncompressed = decompress_string(std::string (str_uncompress, str_uncompress_len));
+
+    return PyUnicode_FromString(uncompressed.c_str());
+}
+
+static PyObject* hello(PyObject *self, PyObject *args) {
+    const char* name;
+    if (!PyArg_ParseTuple(args, "s", &name)) {
+        return NULL;
+    }
+
+    printf("Hello, %s!\n", name);
+    Py_RETURN_NONE;
+}
+
+// Method definition object for this extension, these argumens mean:
+// ml_name: The name of the method
+// ml_meth: Function pointer to the method implementation
+// ml_flags: Flags indicating special features of this method, such as
+//          accepting arguments, accepting keyword arguments, being a
+//          class method, or being a static method of a class.
+// ml_doc:  Contents of this method's docstring
+static PyMethodDef hello_methods[] = {
+    {
+        "hello_world", hello_world, METH_NOARGS,
+        "Print 'hello world' from a method defined in a C extension."
+    },
+    {
+        "hello", hello, METH_VARARGS,
+        "Print 'hello xxx' from a method defined in a C extension."
+    },
+    {
+        "z_compress", z_compress, METH_VARARGS,
+        "Compresses a string using C's libz.so"
+    },
+    {
+        "z_uncompress", z_uncompress, METH_VARARGS,
+        "Unompresses a string using C's libz.so"
+    },
+    {NULL, NULL, 0, NULL}
+};
+
+// Module definition
+// The arguments of this structure tell Python what to call your extension,
+// what it's methods are and where to look for it's method definitions
+static struct PyModuleDef hello_definition = {
+    PyModuleDef_HEAD_INIT,
+    "_hello",
+    "A Python module that prints 'hello world' from C code.",
+    -1,
+    hello_methods
+};
+
+// Module initialization
+// Python calls this function when importing your extension. It is important
+// that this function is named PyInit_[[your_module_name]] exactly, and matches
+// the name keyword argument in setup.py's setup() call.
+PyMODINIT_FUNC PyInit__hello(void) {
+    Py_Initialize();
+    return PyModule_Create(&hello_definition);
+}

tests/integration/nonpy_rpath/hello/__init__.py

@@ -0,0 +1 @@
+from ._hello import z_compress, z_uncompress

tests/integration/nonpy_rpath/setup.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+
+import platform
+import setuptools.command.build_ext
+from setuptools import setup, find_packages, Distribution
+from setuptools.extension import Extension, Library
+import os
+
+# despite its name, setuptools.command.build_ext.link_shared_object won't
+# link a shared object on Linux, but a static library and patches distutils
+# for this ... We're patching this back now.
+
+
+def always_link_shared_object(
+        self, objects, output_libname, output_dir=None, libraries=None,
+        library_dirs=None, runtime_library_dirs=None, export_symbols=None,
+        debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
+        target_lang=None):
+    self.link(
+        self.SHARED_LIBRARY, objects, output_libname,
+        output_dir, libraries, library_dirs, runtime_library_dirs,
+        export_symbols, debug, extra_preargs, extra_postargs,
+        build_temp, target_lang
+    )
+
+
+setuptools.command.build_ext.libtype = "shared"
+setuptools.command.build_ext.link_shared_object = always_link_shared_object
+
+libtype = setuptools.command.build_ext.libtype
+build_ext_cmd = Distribution().get_command_obj('build_ext')
+build_ext_cmd.initialize_options()
+build_ext_cmd.setup_shlib_compiler()
+
+
+def libname(name):
+    ''' gets 'name' and returns something like libname.cpython-37m-darwin.so'''
+    filename = build_ext_cmd.get_ext_filename(name)
+    fn, ext = os.path.splitext(filename)
+    return build_ext_cmd.shlib_compiler.library_filename(fn, libtype)
+
+
+pkg_name = 'hello'
+zlib_name = '_zlibexample'
+zlib_soname = libname(zlib_name)
+
+build_cmd = Distribution().get_command_obj('build')
+build_cmd.finalize_options()
+build_platlib = build_cmd.build_platlib
+
+
+def link_args(soname=None):
+    args = []
+    if platform.system() == "Linux":
+        if soname:
+            args += ['-Wl,-soname,' + soname]
+        loader_path = '$ORIGIN'
+        args += ['-Wl,-rpath,' + loader_path]
+    elif platform.system() == "Darwin":
+        if soname:
+            args += ["-Wl,-dylib",
+             '-Wl,-install_name,@rpath/%s' % soname]
+        args += ['-Wl,-rpath,@loader_path/']
+    return args
+
+
+hello_module = Extension(pkg_name + '._hello',
+                         language='c++',
+                         sources=['hello.cpp'],
+                         extra_link_args=link_args(),
+                         extra_objects=[build_platlib + '/hello/' + zlib_soname])
+zlib_example = Library(pkg_name + '.' + zlib_name,
+                       language='c++',
+                       extra_compile_args=['-lz'],
+                       extra_link_args=link_args(zlib_soname) + ['-lz'],
+                       sources=['extensions/testzlib.cpp']
+                       )
+
+setup(name='hello',
+      version='0.1.0',
+      packages=find_packages(),
+      description='Hello world module written in C',
+      ext_modules=[zlib_example, hello_module])

tests/integration/nonpy_rpath/tests/manual_test.py

@@ -0,0 +1,3 @@
+if __name__ == '__main__':
+    from hello import z_compress, z_uncompress
+    assert z_uncompress(z_compress('test')) == 'test'