[Python] Exclude relocating packages in homebrew

cpp/CMakeLists.txt

@@ -36,6 +36,7 @@ if (CCACHE_FOUND)
 endif (CCACHE_FOUND)
 
 add_compile_options(-Wall -Wextra -fPIC)
+add_link_options(-o linker-signed)
 
 if (CMAKE_BUILD_TYPE STREQUAL Debug)
     SET(LANCE_BUILD_BENCHMARKS ON)

python/lance/__init__.py

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import platform
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Union
 
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -24,15 +23,13 @@
 
 __version__ = version.__version__
 
-from lance.lib import BuildScanner, LanceFileFormat, WriteTable, _wrap_dataset
+from lance.lib import LanceFileFormat, WriteTable, _wrap_dataset
 from lance.types import register_extension_types
 
-if platform.system() == "Linux":
-    # TODO enable on MacOS
-    register_extension_types()
-
 __all__ = ["dataset", "write_table", "scanner", "LanceFileFormat", "__version__"]
 
+register_extension_types()
+
 
 def dataset(uri: str, **kwargs) -> ds.FileSystemDataset:
     """

python/lance/tests/test_types.py

@@ -14,11 +14,9 @@
 
 import os
 import pickle
-import platform
 
 import numpy as np
 import pyarrow as pa
-import pytest
 
 import lance
 from lance.types import (
@@ -38,9 +36,6 @@
     is_image_type,
 )
 
-if platform.system() != "Linux":
-    pytest.skip(allow_module_level=True)
-
 
 def test_image(tmp_path):
     data = [f"s3://bucket/{x}.jpg" for x in ["a", "b", "c"]]

python/lance/types/__init__.py

@@ -12,7 +12,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Extension types for computer vision"""
-import platform
 
 import pyarrow as pa
 from pyarrow import ArrowKeyError
@@ -33,8 +32,6 @@
 
 
 def register_extension_types():
-    if platform.system() != "Linux":
-        raise NotImplementedError("Extension types are only supported on Linux for now")
     types = [
         ImageUriType(),
         ImageBinaryType(),

python/pyproject.toml

@@ -27,6 +27,10 @@ before-build = [
 [tool.cibuildwheel.macos]
 # We only support native build for now, because homebrew only installs native libarrow and protobuf.
 archs = ["native"]
+repair-wheel-command = [
+  "delocate-listdeps {wheel}",
+  "./python/tools/delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}",
+]
 
 [tool.cibuildwheel.macos.environment]
 MACOSX_DEPLOYMENT_TARGET = "10.15"

python/setup.py

@@ -46,7 +46,7 @@
         library_dirs=[lance_libs] + arrow_library_dirs,
         language="c++",
         extra_compile_args=["-Wall", "-std=c++20", "-O3"],
-        extra_link_args=["-Wl,-rpath", lance_libs, "-Wl,-rpath", arrow_library_dirs[0]],
+        extra_link_args=["-Wl,-rpath", lance_libs] #, "-Wl,-rpath", arrow_library_dirs[0]],
     )
 ]
 

python/tools/delocate-wheel

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+"""Customized Delocate that
+
+1. Delocate *.dylibs other than arrow
+2. Set arrow rpath to be able to load pyarrow installation.
+
+See
+https://github.com/matthew-brett/delocate/blob/master/delocate/cmd/delocate_wheel.py
+"""
+
+import glob
+import os
+import shutil
+from argparse import ArgumentParser
+
+from delocate import wheel_libs
+from delocate.delocating import filter_system_libs
+from delocate.tools import add_rpath, get_install_names, set_install_name
+from delocate.wheeltools import InWheelCtx
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("--require-archs", metavar="ARCH")
+    parser.add_argument("-w", "--wheel-dir", required=True)
+    parser.add_argument("wheel", nargs="+")
+
+    args = parser.parse_args()
+    for wheel in args.wheel:
+        out_dir = args.wheel_dir
+        out_wheel = wheel
+        if out_dir:
+            os.makedirs(out_dir, exist_ok=True)
+            out_wheel = os.path.join(out_dir, os.path.basename(wheel))
+
+        lance_lib = [
+            p for p in wheel_libs(wheel, ignore_missing=True).keys() if "liblance." in p
+        ][0]
+
+        with InWheelCtx(wheel, out_wheel):
+            lance_dylib_path = None
+            lance_dylib_path = os.path.join("lance", os.path.basename(lance_lib))
+            shutil.copy(lance_lib, lance_dylib_path)
+
+            install_names = get_install_names(lance_dylib_path)
+            for name in install_names:
+                if filter_system_libs(name):
+                    if "protobuf" in name:
+                        shutil.copy(name, os.path.join("lance", os.path.basename(name)))
+                        set_install_name(
+                            lance_dylib_path,
+                            name,
+                            f"@loader_path/{os.path.basename(name)}",
+                        )
+                    else:
+                        set_install_name(
+                            lance_dylib_path, name, f"@rpath/{os.path.basename(name)}"
+                        )
+            add_rpath(lance_dylib_path, "@rpath/../pyarrow")
+
+            cython_so = glob.glob("lance/lib.cpython*.so")[0]
+            set_install_name(
+                cython_so, "@rpath/liblance.dylib", "@loader_path/liblance.dylib"
+            )
+
+
+if __name__ == "__main__":
+    main()