diff --git a/Makefile b/Makefile index 4443b748f..a02f79094 100644 --- a/Makefile +++ b/Makefile @@ -116,12 +116,21 @@ tarball: hadoop-jars cp ./java/qfs-access/qfs-access*.jar "tmpreldir/$$tarname/lib/"; fi && \ if ls -1 ./java/hadoop-qfs/hadoop-*.jar > /dev/null 2>&1; then \ cp ./java/hadoop-qfs/hadoop-*.jar "tmpreldir/$$tarname/lib/"; fi && \ + if ls -1 ${BUILD_TYPE}/python-qfs/dist/qfs*.whl > /dev/null 2>&1; then \ + cp ${BUILD_TYPE}/python-qfs/dist/qfs*.whl \ + "tmpreldir/$$tarname/lib/"; fi && \ tar cvfz "$$tarname".tgz -C ./tmpreldir "$$tarname" && \ rm -rf tmpreldir .PHONY: python python: build - cd build/${BUILD_TYPE} && python3 ../../src/cc/access/kfs_setup.py build + cd build/${BUILD_TYPE} && \ + rm -rf python-qfs && mkdir python-qfs && cd python-qfs && \ + ln -s .. qfs && \ + ln -s ../../../src/cc/access/kfs_setup.py setup.py && \ + python3 -m venv .venv && \ + . .venv/bin/activate && python -m pip install build && \ + python -m build -w . .PHONY: mintest mintest: hadoop-jars diff --git a/cmake/Modules/FindJerasure.cmake b/cmake/Modules/FindJerasure.cmake index 53e0faa91..9a4c6f398 100644 --- a/cmake/Modules/FindJerasure.cmake +++ b/cmake/Modules/FindJerasure.cmake @@ -88,15 +88,22 @@ set(JERASURE_STATIC_LIBRARIES ${Jerasure_STATIC_LIB} ${Gf_complete_STATIC_LIB} ) -if(CYGWIN AND NOT QFS_JERASURE_CYGWIN_USE_SHARED_LIBS) - # It appears that on cygwin only static libs are built, and it is - # possible to link client library dll against them. +set(Jerasure_STATIC_LIB_SYSTEMS Darwin Linux FreeBSD) +if(CMAKE_SYSTEM_NAME IN_LIST Jerasure_STATIC_LIB_SYSTEMS OR + (CYGWIN AND NOT QFS_JERASURE_CYGWIN_USE_SHARED_LIBS)) + # For now do not use shared libs as libtool sets absolute library path in + # these, and makes it difficult to use with $ORIGIN on Linux and possibly + # BSD and/or @loader_path on Darvin, and on cygwin only static libs are + # built. + # The libraries objects are build with -fPIC and -DPIC flags and the same + # object are used for both static and shared libs, threfore linking with + # other shared library (qfs_client) should work. set(JERASURE_SHARED_LIBRARIES ${JERASURE_STATIC_LIBRARIES}) else() # Shared library are sym linked, install both sym link and the targets # by using pattern. Allow version suffix that follows library suffix. install (DIRECTORY ${Gf_complete_LIB_DIR} ${Jerasure_LIB_DIR} - DESTINATION lib + LIBRARY DESTINATION lib USE_SOURCE_PERMISSIONS FILES_MATCHING PATTERN "${CMAKE_SHARED_LIBRARY_PREFIX}*${CMAKE_SHARED_LIBRARY_SUFFIX}*" diff --git a/src/cc/access/kfs_setup.py b/src/cc/access/kfs_setup.py index 424b43a5b..02247974a 100644 --- a/src/cc/access/kfs_setup.py +++ b/src/cc/access/kfs_setup.py @@ -35,48 +35,52 @@ # file doc/DeveloperDoc. # -from distutils.core import setup, Extension -import sys import os -import os.path +import sys +from distutils.core import Extension, setup -kfs_access_dir=os.path.dirname(sys.argv[0]) +kfs_access_dir = os.path.dirname(os.path.realpath(sys.argv[0])) -kfsext = Extension( - 'qfs', - include_dirs = [ - os.path.abspath(os.path.join(kfs_access_dir, "..")) - ], - libraries = [ - 'qfs_client', - 'qfs_common', - 'qfs_io', - 'qfs_qcdio', - 'qfs_qcrs' - ], - library_dirs = [ - 'src/cc/libclient', - 'src/cc/common', - 'src/cc/kfsio', - 'src/cc/qcdio', - 'src/cc/qcrs', - ], - runtime_library_dirs = [], - sources = [ - os.path.abspath(os.path.join(kfs_access_dir, "kfs_module_py.cc")) +# The following assumes that QFS was build by running cmake and then +# make install +libs_dir = "lib" +if os.path.exists("qfs"): + # kfs_setup.py symlinked to setup.py and build directory symlinked to qfs. + # QFS shared libraries are packaged into qfs/lib and QFS C extension's run + # time path is set accordingly. + libs_dir = os.path.join("qfs", libs_dir) + setup_extra_args = { + "packages": ["qfs"], + "package_data": {"qfs": ["lib/*.*"]}, + # "ext_package": "qfs", + } + extension_extra_args = { + "runtime_library_dirs": { + "darwin": ["@loader_path/qfs/lib"], + "win32": [], + }.get(sys.platform, ["$ORIGIN/qfs/lib"]) + } +else: + # Old way of invoking kfs_setup.py from build directory. + setup_extra_args = {} + extension_extra_args = {} + +qfs_ext = Extension( + "qfs", + include_dirs=[os.path.abspath(os.path.join(kfs_access_dir, ".."))], + libraries=["qfs_client"], + library_dirs=[libs_dir], + sources=[ + os.path.abspath(os.path.join(kfs_access_dir, "kfs_module_py.cc")) ], + **extension_extra_args, ) -# OSX boost ports typically end up at /opt/local/lib -if sys.platform in ('darwin', 'Darwin'): - kfsext.libraries.append('boost_regex-mt') - kfsext.libraries.append('boost_system-mt') -else: - kfsext.libraries.append('boost_regex') - setup( - name = "qfs", version = "2.5", + name="qfs", + version="2.5", description="QFS client module", author="Blake Lewis and Sriram Rao", - ext_modules = [kfsext] + ext_modules=[qfs_ext], + **setup_extra_args, )