diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 208af610e6..51a1c62855 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -36,6 +36,7 @@ if (CCACHE_FOUND) endif (CCACHE_FOUND) add_compile_options(-Wall -Wextra -fPIC) +add_link_options(-o linker-signed) if (CMAKE_BUILD_TYPE STREQUAL Debug) SET(LANCE_BUILD_BENCHMARKS ON) diff --git a/python/lance/__init__.py b/python/lance/__init__.py index 8fd9a4700a..f22fc3d48a 100644 --- a/python/lance/__init__.py +++ b/python/lance/__init__.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import platform from pathlib import Path -from typing import List, Optional, Union +from typing import Union import pyarrow as pa import pyarrow.compute as pc @@ -24,15 +23,13 @@ __version__ = version.__version__ -from lance.lib import BuildScanner, LanceFileFormat, WriteTable, _wrap_dataset +from lance.lib import LanceFileFormat, WriteTable, _wrap_dataset from lance.types import register_extension_types -if platform.system() == "Linux": - # TODO enable on MacOS - register_extension_types() - __all__ = ["dataset", "write_table", "scanner", "LanceFileFormat", "__version__"] +register_extension_types() + def dataset(uri: str, **kwargs) -> ds.FileSystemDataset: """ diff --git a/python/lance/tests/test_types.py b/python/lance/tests/test_types.py index c0136545ef..bccecfd747 100644 --- a/python/lance/tests/test_types.py +++ b/python/lance/tests/test_types.py @@ -14,11 +14,9 @@ import os import pickle -import platform import numpy as np import pyarrow as pa -import pytest import lance from lance.types import ( @@ -38,9 +36,6 @@ is_image_type, ) -if platform.system() != "Linux": - pytest.skip(allow_module_level=True) - def test_image(tmp_path): data = [f"s3://bucket/{x}.jpg" for x in ["a", "b", "c"]] diff --git a/python/lance/types/__init__.py b/python/lance/types/__init__.py index 1ad16c9f82..646bc39930 100644 --- a/python/lance/types/__init__.py +++ b/python/lance/types/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Extension types for computer vision""" -import platform import pyarrow as pa from pyarrow import ArrowKeyError @@ -33,8 +32,6 @@ def register_extension_types(): - if platform.system() != "Linux": - raise NotImplementedError("Extension types are only supported on Linux for now") types = [ ImageUriType(), ImageBinaryType(), diff --git a/python/pyproject.toml b/python/pyproject.toml index cf6d57bba3..f9bf8ae917 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -27,6 +27,10 @@ before-build = [ [tool.cibuildwheel.macos] # We only support native build for now, because homebrew only installs native libarrow and protobuf. archs = ["native"] +repair-wheel-command = [ + "delocate-listdeps {wheel}", + "./python/tools/delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}", +] [tool.cibuildwheel.macos.environment] MACOSX_DEPLOYMENT_TARGET = "10.15" diff --git a/python/setup.py b/python/setup.py index 8a28ee1e11..3158edbb39 100644 --- a/python/setup.py +++ b/python/setup.py @@ -46,7 +46,7 @@ library_dirs=[lance_libs] + arrow_library_dirs, language="c++", extra_compile_args=["-Wall", "-std=c++20", "-O3"], - extra_link_args=["-Wl,-rpath", lance_libs, "-Wl,-rpath", arrow_library_dirs[0]], + extra_link_args=["-Wl,-rpath", lance_libs] #, "-Wl,-rpath", arrow_library_dirs[0]], ) ] diff --git a/python/tools/delocate-wheel b/python/tools/delocate-wheel new file mode 100755 index 0000000000..98ab9439b1 --- /dev/null +++ b/python/tools/delocate-wheel @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +"""Customized Delocate that + +1. Delocate *.dylibs other than arrow +2. Set arrow rpath to be able to load pyarrow installation. + +See +https://github.com/matthew-brett/delocate/blob/master/delocate/cmd/delocate_wheel.py +""" + +import glob +import os +import shutil +from argparse import ArgumentParser + +from delocate import wheel_libs +from delocate.delocating import filter_system_libs +from delocate.tools import add_rpath, get_install_names, set_install_name +from delocate.wheeltools import InWheelCtx + + +def main(): + parser = ArgumentParser() + parser.add_argument("--require-archs", metavar="ARCH") + parser.add_argument("-w", "--wheel-dir", required=True) + parser.add_argument("wheel", nargs="+") + + args = parser.parse_args() + for wheel in args.wheel: + out_dir = args.wheel_dir + out_wheel = wheel + if out_dir: + os.makedirs(out_dir, exist_ok=True) + out_wheel = os.path.join(out_dir, os.path.basename(wheel)) + + lance_lib = [ + p for p in wheel_libs(wheel, ignore_missing=True).keys() if "liblance." in p + ][0] + + with InWheelCtx(wheel, out_wheel): + lance_dylib_path = None + lance_dylib_path = os.path.join("lance", os.path.basename(lance_lib)) + shutil.copy(lance_lib, lance_dylib_path) + + install_names = get_install_names(lance_dylib_path) + for name in install_names: + if filter_system_libs(name): + if "protobuf" in name: + shutil.copy(name, os.path.join("lance", os.path.basename(name))) + set_install_name( + lance_dylib_path, + name, + f"@loader_path/{os.path.basename(name)}", + ) + else: + set_install_name( + lance_dylib_path, name, f"@rpath/{os.path.basename(name)}" + ) + add_rpath(lance_dylib_path, "@rpath/../pyarrow") + + cython_so = glob.glob("lance/lib.cpython*.so")[0] + set_install_name( + cython_so, "@rpath/liblance.dylib", "@loader_path/liblance.dylib" + ) + + +if __name__ == "__main__": + main()