From 41c9768dec0027184c2e47aea6a78e7fe3670dea Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 23 Jan 2017 23:04:56 -0800 Subject: [PATCH 01/44] NUP-2320 Work-in-progress: separation of concerns. Still static lib. --- src/CMakeLists.txt | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b74c6b36d..e5ce803014 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -387,7 +387,6 @@ set(src_nupiccore_srcs set(src_lib_static_nupiccore_srcs ${src_capnp_generated_srcs} - ${src_py_support_files} ${src_nupiccore_srcs}) set(src_lib_static_nupiccore_compile_flags @@ -416,6 +415,31 @@ if(${NUPIC_IWYU}) endif() +# +# Create py_support "object" library that contains python support object files +# for linking into python extensions. +# +set(src_lib_py_support_objects py_support_objects) + +add_library(${src_lib_py_support_objects} OBJECT + ${src_py_support_files}) +add_dependencies(${src_lib_py_support_objects} + ${src_lib_static_nupiccore_solo}) +set_target_properties(${src_lib_py_support_objects} PROPERTIES COMPILE_FLAGS + ${src_lib_static_nupiccore_compile_flags}) +if(${NUPIC_IWYU}) + # TODO: Create a target that doesn't include the generated capnp schema files + # since we don't want to run iwyu on them and iwyu can only be applied to + # CMake targets (not source files directly). + set_target_properties(${src_lib_py_support_objects} + PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) +endif() + +# We use this generator value to add the python support objects to the +# source lists of the modules that rely on it. +set(src_py_support_object_list_gen + "$") + # # Create the nupic_core "combined" static library by merging nupic_core_solo # with our external static libraries @@ -472,7 +496,9 @@ message(STATUS "src_common_test_exe_libs = ${src_common_test_exe_libs}") # Setup test_cpp_region # set(src_executable_cppregiontest cpp_region_test) -add_executable(${src_executable_cppregiontest} test/integration/CppRegionTest.cpp) +add_executable(${src_executable_cppregiontest} + test/integration/CppRegionTest.cpp + ${src_py_support_object_list_gen}) target_link_libraries(${src_executable_cppregiontest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_cppregiontest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_cppregiontest} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") @@ -486,7 +512,9 @@ add_custom_target(tests_cpp_region # Setup test_py_region # set(src_executable_pyregiontest py_region_test) -add_executable(${src_executable_pyregiontest} test/integration/PyRegionTest.cpp) +add_executable(${src_executable_pyregiontest} + test/integration/PyRegionTest.cpp + ${src_py_support_object_list_gen}) target_link_libraries(${src_executable_pyregiontest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_pyregiontest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -520,7 +548,9 @@ add_custom_target(tests_connections_performance # Setup helloregion example # set(src_executable_helloregion helloregion) -add_executable(${src_executable_helloregion} examples/regions/HelloRegions.cpp) +add_executable(${src_executable_helloregion} + examples/regions/HelloRegions.cpp + ${src_py_support_object_list_gen}) target_link_libraries(${src_executable_helloregion} ${src_common_test_exe_libs}) set_target_properties(${src_executable_helloregion} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -555,6 +585,7 @@ set_target_properties(${src_executable_hellosptp} # set(src_executable_gtests unit_tests) add_executable(${src_executable_gtests} + ${src_py_support_object_list_gen} test/unit/algorithms/AnomalyTest.cpp test/unit/algorithms/CondProbTableTest.cpp test/unit/algorithms/ConnectionsTest.cpp @@ -769,7 +800,9 @@ if (NUPIC_BUILD_PYEXT_MODULES) # # Note: swig_add_module outputs ${swig_generated_file_fullname} - swig_add_module(${MODULE_NAME} python ${source_interface_file}) + swig_add_module(${MODULE_NAME} python + ${source_interface_file} + ${src_py_support_object_list_gen}) set_source_files_properties( ${swig_generated_file_fullname} PROPERTIES From 9a9a0b5dc72a39d33e3e074fae1786d2144d272d Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 24 Jan 2017 14:53:37 -0800 Subject: [PATCH 02/44] NUP-2320 Created nupic_core shared lib, but unit_tests fails to link. --- CMakeLists.txt | 2 +- external/CMakeLists.txt | 2 +- src/CMakeLists.txt | 109 ++++++++++++++++++++++-------------- src/NupicLibraryUtils.cmake | 2 +- 4 files changed, 70 insertions(+), 45 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8cf0c2c0df..c8b3f6ef38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ # http://numenta.org/licenses/ # ----------------------------------------------------------------------------- -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) project(nupic_core_main CXX) set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}") diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 62d6d3b3df..696307132e 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -19,7 +19,7 @@ # http://numenta.org/licenses/ # ----------------------------------------------------------------------------- -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) project(nupic_core_main CXX) set(CMAKE_VERBOSE_MAKEFILE OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e5ce803014..4fa06396f6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,7 +19,7 @@ # http://numenta.org/licenses/ # ----------------------------------------------------------------------------- -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) project(nupic_core CXX) set_property(GLOBAL PROPERTY USE_FOLDERS ON) @@ -302,15 +302,16 @@ message(STATUS "src_common_os_libs = ${src_common_os_libs}") # -# Setup nupic_core_solo static library, consisting of our own sources; +# Setup a library of objects, generated from our own sources without py support # -# this is an intermediate archive that will be merged with external static -# libraries in a subsequent step to produce the nupic_core "combined" static -# library. -# -set(src_lib_static_nupiccore_solo nupic_core_solo) +set(src_lib_object_nupiccore_internals nupic_core_internal_objects) + +# We use this generator value to add the nupic core internal objects to the +# source lists of the modules that rely on them. +set(src_nupiccore_internal_object_list_gen + "$") -set(src_nupiccore_srcs +set(src_nupiccore_internal_srcs nupic/algorithms/Anomaly.cpp nupic/algorithms/BitHistory.cpp nupic/algorithms/Cell.cpp @@ -385,48 +386,46 @@ set(src_nupiccore_srcs nupic/utils/TRandom.cpp nupic/utils/Watcher.cpp) -set(src_lib_static_nupiccore_srcs - ${src_capnp_generated_srcs} - ${src_nupiccore_srcs}) +list(APPEND src_nupiccore_internal_srcs ${src_capnp_generated_srcs}) -set(src_lib_static_nupiccore_compile_flags +set(src_lib_object_nupiccore_internals_compile_flags "${src_compile_flags} -I${src_numpy_core}/include") message(STATUS "src_compile_flags = ${src_compile_flags}") -message(STATUS "src_lib_static_nupiccore_compile_flags = ${src_lib_static_nupiccore_compile_flags}") +message(STATUS "src_lib_object_nupiccore_internals_compile_flags = ${src_lib_object_nupiccore_internals_compile_flags}") -add_library(${src_lib_static_nupiccore_solo} STATIC - ${src_lib_static_nupiccore_srcs}) +add_library(${src_lib_object_nupiccore_internals} OBJECT + ${src_nupiccore_internal_srcs}) # nupic_core_solo sources depend on headers installed by these external projects -add_dependencies(${src_lib_static_nupiccore_solo} +add_dependencies(${src_lib_object_nupiccore_internals} ${YAML_CPP_STATIC_LIB_TARGET} ${YAML_STATIC_LIB_TARGET} ${APR1_STATIC_LIB_TARGET} ${APRUTIL1_STATIC_LIB_TARGET} ${Z_STATIC_LIB_TARGET}) -set_target_properties(${src_lib_static_nupiccore_solo} PROPERTIES COMPILE_FLAGS - ${src_lib_static_nupiccore_compile_flags}) +set_target_properties(${src_lib_object_nupiccore_internals} PROPERTIES COMPILE_FLAGS + ${src_lib_object_nupiccore_internals_compile_flags}) if(${NUPIC_IWYU}) # TODO: Create a target that doesn't include the generated capnp schema files # since we don't want to run iwyu on them and iwyu can only be applied to # CMake targets (not source files directly). - set_target_properties(${src_lib_static_nupiccore_solo} + set_target_properties(${src_lib_object_nupiccore_internals} PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() # -# Create py_support "object" library that contains python support object files -# for linking into python extensions. +# Create py_support "object" library that generates python support object files +# for linking into python extensions and c++ tests that need it. # set(src_lib_py_support_objects py_support_objects) add_library(${src_lib_py_support_objects} OBJECT ${src_py_support_files}) add_dependencies(${src_lib_py_support_objects} - ${src_lib_static_nupiccore_solo}) + ${src_lib_object_nupiccore_internals}) set_target_properties(${src_lib_py_support_objects} PROPERTIES COMPILE_FLAGS - ${src_lib_static_nupiccore_compile_flags}) + ${src_lib_object_nupiccore_internals_compile_flags}) if(${NUPIC_IWYU}) # TODO: Create a target that doesn't include the generated capnp schema files # since we don't want to run iwyu on them and iwyu can only be applied to @@ -436,18 +435,16 @@ if(${NUPIC_IWYU}) endif() # We use this generator value to add the python support objects to the -# source lists of the modules that rely on it. +# source lists of the modules that rely on them. set(src_py_support_object_list_gen "$") # -# Create the nupic_core "combined" static library by merging nupic_core_solo -# with our external static libraries +# Create the nupic_core shared library containing nupic core internals (sans +# python support) and our externals # -include(src/NupicLibraryUtils) # for MERGE_STATIC_LIBRARIES - -set(src_lib_static_nupiccore_combined nupic_core) +set(src_lib_shared_nupiccore nupic_core) set(src_external_static_libs ${YAML_CPP_STATIC_LIB_TARGET} @@ -456,29 +453,57 @@ set(src_external_static_libs ${APRUTIL1_STATIC_LIB_TARGET} ${Z_STATIC_LIB_TARGET}) -# Add capnproto static lib when building nupic_core static lib without python -# extensions. The extensions have their own logic governing whether to include -# it. if (NOT NUPIC_BUILD_PYEXT_MODULES) + # When building a standalone shared lib that will run without our python + # extensions, add capnproto static lib. Our python extensions have their own + # logic governing whether to include capnproto. list(APPEND src_external_static_libs ${CAPNP_STATIC_LIB_TARGET}) endif() -set(src_combined_nupiccore_source_archives - ${src_lib_static_nupiccore_solo} - ${src_external_static_libs}) +set(src_lib_shared_nupiccore_link_libraries ${src_external_static_libs}) +if (NOT NUPIC_BUILD_PYEXT_MODULES) + # When building a standalone shared lib that will run outside the python + # environment, add dependency on python lib to resolve python symbol + # references (e.g., from PyRegion) + list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) +endif() + +add_library(${src_lib_shared_nupiccore} SHARED + ${src_nupiccore_internal_object_list_gen}) -# Create a top-level library target for the combined static lib -merge_static_libraries(${src_lib_static_nupiccore_combined} - "${src_combined_nupiccore_source_archives}") +target_link_libraries(${src_lib_shared_nupiccore} + ${src_lib_shared_nupiccore_link_libraries}) +if (NUPIC_BUILD_PYEXT_MODULES) + # When building a shared lib for use by our pythone extensions, allow building + # without resolving all symbols. This addresses resolution of python symbols + # that will be provided by the python runtime as well as capnproto symbols + # that will be provided by pycapnp. + set_target_properties(${src_lib_shared_nupiccore} PROPERTIES LINK_FLAGS + "${PYEXT_LINKER_FLAGS_OPTIMIZED}") +else() + # When building a standalone shared library, specifically don't allow link + # with undefined symbols + set_target_properties(${src_lib_shared_nupiccore} PROPERTIES LINK_FLAGS + "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +endif() + +if (APPLE) + set_target_properties(${src_lib_shared_nupiccore} PROPERTIES MACOSX_RPATH ON) +endif() + +if (NUPIC_BUILD_PYEXT_MODULES AND PY_EXTENSIONS_DIR) + set_target_properties(${src_lib_shared_nupiccore} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${PY_EXTENSIONS_DIR}) +endif() # -# Build tests of the nupic_core "combined" static library +# Build tests of the nupic_core shared library # # Common libs for test executables set(src_common_test_exe_libs - ${src_lib_static_nupiccore_combined} + ${src_lib_shared_nupiccore} ${PYTHON_LIBRARIES} ${src_common_os_libs}) @@ -729,7 +754,7 @@ if (NUPIC_BUILD_PYEXT_MODULES) # python proxy modules, thus avoiding the conflict from executing the methods # in our own compilation of capnproto on objects created by pycapnp's. set(_SRC_SWIG_LINK_LIBRARIES - ${src_lib_static_nupiccore_combined} + ${src_lib_shared_nupiccore} ${src_common_os_libs}) # Common dependencies for our python extensions for use with @@ -882,7 +907,7 @@ endif() # NUPIC_BUILD_PYEXT_MODULES # Install targets into CMAKE_INSTALL_PREFIX # install(TARGETS - ${src_lib_static_nupiccore_combined} + ${src_lib_shared_nupiccore} ${src_lib_static_gtest} ${src_executable_helloregion} ${src_executable_cppregiontest} diff --git a/src/NupicLibraryUtils.cmake b/src/NupicLibraryUtils.cmake index c997483fd4..34feebe527 100644 --- a/src/NupicLibraryUtils.cmake +++ b/src/NupicLibraryUtils.cmake @@ -21,7 +21,7 @@ # Utilities for manipulating libraries -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) project(nupic_core_library_utils CXX) From cfe1a38d4c5dcf51cd900ead27af37d4b0daca9d Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 24 Jan 2017 18:01:18 -0800 Subject: [PATCH 03/44] NUPC-2320 Converted to shared library, but on OS X Sierra, getting error from `import nupic.bindings.engine_internal`: ImportError: dlopen(/Users/vkruglikov/Library/Python/2.7/lib/python/site-packages/nupic/bindings/_engine_internal.so, 2): Library not loaded: libnupic_core.dylib Referenced from: /Users/vkruglikov/Library/Python/2.7/lib/python/site-packages/nupic/bindings/_engine_internal.so Reason: unsafe use of relative rpath libnupic_core.dylib in /Users/vkruglikov/Library/Python/2.7/lib/python/site-packages/nupic/bindings/_engine_internal.so with restricted binary In [3]: import nupic.bindings.engine_internal --- CommonCompilerConfig.cmake | 30 ++++++---- bindings/py/setup.py | 4 +- src/CMakeLists.txt | 117 ++++++++++++++++++------------------- 3 files changed, 77 insertions(+), 74 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index 69a699216d..783d019e61 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -66,9 +66,12 @@ # and shared libraries (DLLs) with optimizations that are # compatible with INTERNAL_CXX_FLAGS_OPTIMIZED # +# PYEXT_CXX_FLAGS_OPTIMIZED: string of C++ flags with explicit optimization +# flags for compiling python extension sources. +# # PYEXT_LINKER_FLAGS_OPTIMIZED: string of linker flags for linking python extension # shared libraries (DLLs) with optimizations that are -# compatible with EXTERNAL_CXX_FLAGS_OPTIMIZED. +# compatible with PYEXT_CXX_FLAGS_OPTIMIZED. # # CMAKE_AR: Name of archiving tool (ar) for static libraries. See cmake documentation # @@ -96,6 +99,7 @@ set(COMMON_COMPILER_DEFINITIONS_STR) set(INTERNAL_CXX_FLAGS_OPTIMIZED) set(INTERNAL_LINKER_FLAGS_OPTIMIZED) +set(PYEXT_CXX_FLAGS_OPTIMIZED) set(PYEXT_LINKER_FLAGS_OPTIMIZED) set(EXTERNAL_C_FLAGS_UNOPTIMIZED) @@ -237,6 +241,8 @@ set(cxx_flags_unoptimized "") set(shared_linker_flags_unoptimized "") set(fail_link_on_undefined_symbols_flags "") set(allow_link_with_undefined_symbols_flags "") +set(cxx_hidden_vis_compile_flags "") +set(shared_hidden_vis_compile_flags "") if(${CMAKE_CXX_COMPILER_ID} STREQUAL "MSVC") # MS Visual C @@ -252,12 +258,10 @@ else() # LLVM Clang / Gnu GCC set(cxx_flags_unoptimized "${cxx_flags_unoptimized} ${stdlib_cxx} -std=c++11") - if (${NUPIC_BUILD_PYEXT_MODULES}) - # Hide all symbols in DLLs except the ones with explicit visibility; - # see https://gcc.gnu.org/wiki/Visibility - set(cxx_flags_unoptimized "${cxx_flags_unoptimized} -fvisibility-inlines-hidden") - set(shared_compile_flags "${shared_compile_flags} -fvisibility=hidden") - endif() + # Hide all symbols in DLLs except the ones with explicit visibility; + # see https://gcc.gnu.org/wiki/Visibility + set(cxx_hidden_vis_compile_flags "${cxx_hidden_vis_compile_flags} -fvisibility-inlines-hidden") + set(shared_hidden_vis_compile_flags "${shared_hidden_vis_compile_flags} -fvisibility=hidden") set(shared_compile_flags "${shared_compile_flags} ${stdlib_common} -fdiagnostics-show-option") set (internal_compiler_warning_flags "${internal_compiler_warning_flags} -Werror -Wextra -Wreturn-type -Wunused -Wno-unused-variable -Wno-unused-parameter -Wno-missing-field-initializers") @@ -370,16 +374,19 @@ set(INTERNAL_LINKER_FLAGS_OPTIMIZED "${complete_linker_flags_unoptimized} ${opti set(EXTERNAL_C_FLAGS_UNOPTIMIZED "${build_type_specific_compile_flags} ${shared_compile_flags} ${external_compiler_warning_flags}") set(EXTERNAL_C_FLAGS_OPTIMIZED "${EXTERNAL_C_FLAGS_UNOPTIMIZED} ${optimization_flags_cc}") -set(PYEXT_LINKER_FLAGS_OPTIMIZED "${build_type_specific_linker_flags} ${shared_linker_flags_unoptimized}") -set(PYEXT_LINKER_FLAGS_OPTIMIZED "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${optimization_flags_lt}") -set(PYEXT_LINKER_FLAGS_OPTIMIZED "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${allow_link_with_undefined_symbols_flags}") - set(EXTERNAL_CXX_FLAGS_UNOPTIMIZED "${build_type_specific_compile_flags} ${shared_compile_flags} ${external_compiler_warning_flags} ${cxx_flags_unoptimized}") set(EXTERNAL_CXX_FLAGS_OPTIMIZED "${EXTERNAL_CXX_FLAGS_UNOPTIMIZED} ${optimization_flags_cc}") set(EXTERNAL_LINKER_FLAGS_UNOPTIMIZED "${complete_linker_flags_unoptimized}") set(EXTERNAL_LINKER_FLAGS_OPTIMIZED "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +set(PYEXT_CXX_FLAGS_OPTIMIZED "${EXTERNAL_CXX_FLAGS_OPTIMIZED} ${shared_hidden_vis_compile_flags}") +set(PYEXT_CXX_FLAGS_OPTIMIZED "${PYEXT_CXX_FLAGS_OPTIMIZED} ${cxx_hidden_vis_compile_flags}") + +set(PYEXT_LINKER_FLAGS_OPTIMIZED "${build_type_specific_linker_flags} ${shared_linker_flags_unoptimized}") +set(PYEXT_LINKER_FLAGS_OPTIMIZED "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${optimization_flags_lt}") +set(PYEXT_LINKER_FLAGS_OPTIMIZED "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${allow_link_with_undefined_symbols_flags}") + # # Provide a string variant of the COMMON_COMPILER_DEFINITIONS list @@ -393,6 +400,7 @@ message(STATUS "INTERNAL_CXX_FLAGS_OPTIMIZED=${INTERNAL_CXX_FLAGS_OPTIMIZED}") message(STATUS "INTERNAL_LINKER_FLAGS_OPTIMIZED=${INTERNAL_LINKER_FLAGS_OPTIMIZED}") message(STATUS "EXTERNAL_C_FLAGS_UNOPTIMIZED=${EXTERNAL_C_FLAGS_UNOPTIMIZED}") message(STATUS "EXTERNAL_C_FLAGS_OPTIMIZED=${EXTERNAL_C_FLAGS_OPTIMIZED}") +message(STATUS "PYEXT_CXX_FLAGS_OPTIMIZED=${PYEXT_CXX_FLAGS_OPTIMIZED}") message(STATUS "PYEXT_LINKER_FLAGS_OPTIMIZED=${PYEXT_LINKER_FLAGS_OPTIMIZED}") message(STATUS "EXTERNAL_CXX_FLAGS_UNOPTIMIZED=${EXTERNAL_CXX_FLAGS_UNOPTIMIZED}") message(STATUS "EXTERNAL_CXX_FLAGS_OPTIMIZED=${EXTERNAL_CXX_FLAGS_OPTIMIZED}") diff --git a/bindings/py/setup.py b/bindings/py/setup.py index 1d85adea55..058ab7448b 100644 --- a/bindings/py/setup.py +++ b/bindings/py/setup.py @@ -213,13 +213,13 @@ def generateExtensions(): # This distribution contains platform-specific C++ libraries, but they are not # built with distutils. So we must create a dummy Extension object so when we # create a binary file it knows to make it platform-specific. - ext_modules=[Extension('nupic.dummy', sources = ['dummy.c'])], + ext_modules=[Extension('nupic.dummy', sources = ['dummy.c'])], namespace_packages=["nupic"], install_requires=findRequirements(platform), packages=find_packages(), package_data={ "nupic.proto": ["*.capnp"], - "nupic.bindings": ["*.so", "*.pyd"], + "nupic.bindings": ["*.so", "*.dylib", "*.pyd"], }, extras_require = {"capnp": ["pycapnp==0.5.8"]}, zip_safe=False, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4fa06396f6..e6ce48b32c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,7 +60,7 @@ endif() # set(src_compile_flags "${INTERNAL_CXX_FLAGS_OPTIMIZED}") -set(src_swig_generated_file_compile_flags "${EXTERNAL_CXX_FLAGS_OPTIMIZED}") +set(src_swig_generated_file_compile_flags "${PYEXT_CXX_FLAGS_OPTIMIZED}") if(MINGW) # This is for GCC 4.8.x @@ -302,15 +302,8 @@ message(STATUS "src_common_os_libs = ${src_common_os_libs}") # -# Setup a library of objects, generated from our own sources without py support +# Setup a list of our own sources without py support # -set(src_lib_object_nupiccore_internals nupic_core_internal_objects) - -# We use this generator value to add the nupic core internal objects to the -# source lists of the modules that rely on them. -set(src_nupiccore_internal_object_list_gen - "$") - set(src_nupiccore_internal_srcs nupic/algorithms/Anomaly.cpp nupic/algorithms/BitHistory.cpp @@ -388,56 +381,11 @@ set(src_nupiccore_internal_srcs list(APPEND src_nupiccore_internal_srcs ${src_capnp_generated_srcs}) -set(src_lib_object_nupiccore_internals_compile_flags +set(src_nupiccore_internals_compile_flags "${src_compile_flags} -I${src_numpy_core}/include") message(STATUS "src_compile_flags = ${src_compile_flags}") -message(STATUS "src_lib_object_nupiccore_internals_compile_flags = ${src_lib_object_nupiccore_internals_compile_flags}") - -add_library(${src_lib_object_nupiccore_internals} OBJECT - ${src_nupiccore_internal_srcs}) -# nupic_core_solo sources depend on headers installed by these external projects -add_dependencies(${src_lib_object_nupiccore_internals} - ${YAML_CPP_STATIC_LIB_TARGET} - ${YAML_STATIC_LIB_TARGET} - ${APR1_STATIC_LIB_TARGET} - ${APRUTIL1_STATIC_LIB_TARGET} - ${Z_STATIC_LIB_TARGET}) -set_target_properties(${src_lib_object_nupiccore_internals} PROPERTIES COMPILE_FLAGS - ${src_lib_object_nupiccore_internals_compile_flags}) -if(${NUPIC_IWYU}) - # TODO: Create a target that doesn't include the generated capnp schema files - # since we don't want to run iwyu on them and iwyu can only be applied to - # CMake targets (not source files directly). - set_target_properties(${src_lib_object_nupiccore_internals} - PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() - - -# -# Create py_support "object" library that generates python support object files -# for linking into python extensions and c++ tests that need it. -# -set(src_lib_py_support_objects py_support_objects) - -add_library(${src_lib_py_support_objects} OBJECT - ${src_py_support_files}) -add_dependencies(${src_lib_py_support_objects} - ${src_lib_object_nupiccore_internals}) -set_target_properties(${src_lib_py_support_objects} PROPERTIES COMPILE_FLAGS - ${src_lib_object_nupiccore_internals_compile_flags}) -if(${NUPIC_IWYU}) - # TODO: Create a target that doesn't include the generated capnp schema files - # since we don't want to run iwyu on them and iwyu can only be applied to - # CMake targets (not source files directly). - set_target_properties(${src_lib_py_support_objects} - PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() - -# We use this generator value to add the python support objects to the -# source lists of the modules that rely on them. -set(src_py_support_object_list_gen - "$") +message(STATUS "src_nupiccore_internals_compile_flags = ${src_nupiccore_internals_compile_flags}") # # Create the nupic_core shared library containing nupic core internals (sans @@ -468,12 +416,21 @@ if (NOT NUPIC_BUILD_PYEXT_MODULES) list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) endif() -add_library(${src_lib_shared_nupiccore} SHARED - ${src_nupiccore_internal_object_list_gen}) +add_library(${src_lib_shared_nupiccore} SHARED ${src_nupiccore_internal_srcs}) target_link_libraries(${src_lib_shared_nupiccore} ${src_lib_shared_nupiccore_link_libraries}) +set_target_properties(${src_lib_shared_nupiccore} PROPERTIES COMPILE_FLAGS + ${src_nupiccore_internals_compile_flags}) +if(${NUPIC_IWYU}) + # TODO: Create a target that doesn't include the generated capnp schema files + # since we don't want to run iwyu on them and iwyu can only be applied to + # CMake targets (not source files directly). + set_target_properties(${src_lib_shared_nupiccore} + PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) +endif() + if (NUPIC_BUILD_PYEXT_MODULES) # When building a shared lib for use by our pythone extensions, allow building # without resolving all symbols. This addresses resolution of python symbols @@ -489,7 +446,7 @@ else() endif() if (APPLE) - set_target_properties(${src_lib_shared_nupiccore} PROPERTIES MACOSX_RPATH ON) + set_target_properties(${src_lib_shared_nupiccore} PROPERTIES MACOSX_RPATH OFF) endif() if (NUPIC_BUILD_PYEXT_MODULES AND PY_EXTENSIONS_DIR) @@ -497,6 +454,39 @@ if (NUPIC_BUILD_PYEXT_MODULES AND PY_EXTENSIONS_DIR) LIBRARY_OUTPUT_DIRECTORY ${PY_EXTENSIONS_DIR}) endif() + + +# +# Create py_support "object" library that generates python support object files +# for linking into python extensions and c++ tests that need it. +# +set(src_lib_py_support_objects py_support_objects) + +add_library(${src_lib_py_support_objects} OBJECT + ${src_py_support_files}) + +# We rely on artifacts from building of nupic_core shared lib (e.g., +# capnp schema C++ headers) +add_dependencies(${src_lib_py_support_objects} + ${src_lib_shared_nupiccore}) + +set_target_properties(${src_lib_py_support_objects} PROPERTIES COMPILE_FLAGS + ${src_nupiccore_internals_compile_flags}) +if(${NUPIC_IWYU}) + # TODO: Create a target that doesn't include the generated capnp schema files + # since we don't want to run iwyu on them and iwyu can only be applied to + # CMake targets (not source files directly). + set_target_properties(${src_lib_py_support_objects} + PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) +endif() + +# We use this generator value to add the python support objects to the +# source lists of the modules that rely on them. +set(src_py_support_object_list_gen + "$") + + + # # Build tests of the nupic_core shared library # @@ -701,8 +691,6 @@ if (NUPIC_BUILD_PYEXT_MODULES) # Make sure the directory exists for the generated C++ files. file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/nupic/bindings) - # TODO ZZZ set COMPILE_FLAGS on swig targets - # SWIG options from: # https://github.com/swig/swig/blob/master/Source/Modules/python.cxx#L111 set(src_swig_flags @@ -920,6 +908,13 @@ install(TARGETS LIBRARY DESTINATION lib ARCHIVE DESTINATION lib) +# Install a copy of nupic_core shared lib into bin to facilitate running tests +# in bin that depend on it. +install(TARGETS + ${src_lib_shared_nupiccore} + LIBRARY DESTINATION bin) + + install(DIRECTORY nupic/ DESTINATION include/nupic FILES_MATCHING PATTERN "*.h*" PATTERN "*.hpp.in" EXCLUDE) From 8221cd19f055aff749c356f5c93273d269ade1ab Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 10:19:06 -0800 Subject: [PATCH 04/44] fixup --- src/CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e6ce48b32c..b44b2c70ab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -462,6 +462,11 @@ endif() # set(src_lib_py_support_objects py_support_objects) +# We use this generator value to add the python support objects to the +# source lists of the modules that rely on them. +set(src_py_support_object_list_gen + "$") + add_library(${src_lib_py_support_objects} OBJECT ${src_py_support_files}) @@ -480,11 +485,6 @@ if(${NUPIC_IWYU}) PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() -# We use this generator value to add the python support objects to the -# source lists of the modules that rely on them. -set(src_py_support_object_list_gen - "$") - # @@ -546,7 +546,8 @@ add_custom_target(tests_py_region # set(src_executable_connectionsperformancetest connections_performance_test) add_executable(${src_executable_connectionsperformancetest} - test/integration/ConnectionsPerformanceTest.cpp) + test/integration/ConnectionsPerformanceTest.cpp + ${src_py_support_object_list_gen}) target_link_libraries(${src_executable_connectionsperformancetest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_connectionsperformancetest} From e6220401a9eeb4264db9a891ebf9e1f7921210dc Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 11:20:55 -0800 Subject: [PATCH 05/44] NUP-2320 Add python support objects to tests builds. NUP-2320 Fix installation of nupic_core dll into the prefix/bin directory so that it works on Windows as well, since there it's treated as RUNTIME instead of LIBRARY. --- src/CMakeLists.txt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b44b2c70ab..cb71222592 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -577,7 +577,9 @@ set_target_properties(${src_executable_helloregion} # Setup prototest example # set(src_executable_prototest prototest) -add_executable(${src_executable_prototest} examples/prototest.cpp) +add_executable(${src_executable_prototest} + examples/prototest.cpp + ${src_py_support_object_list_gen}) target_link_libraries(${src_executable_prototest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_prototest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -588,7 +590,9 @@ set_target_properties(${src_executable_prototest} # Setup HelloSP_TP example # set(src_executable_hellosptp hello_sp_tp) -add_executable(${src_executable_hellosptp} examples/algorithms/HelloSP_TP.cpp) +add_executable(${src_executable_hellosptp} + examples/algorithms/HelloSP_TP.cpp + ${src_py_support_object_list_gen}) target_link_libraries(${src_executable_hellosptp} ${src_common_test_exe_libs}) set_target_properties(${src_executable_hellosptp} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -911,9 +915,8 @@ install(TARGETS # Install a copy of nupic_core shared lib into bin to facilitate running tests # in bin that depend on it. -install(TARGETS - ${src_lib_shared_nupiccore} - LIBRARY DESTINATION bin) +install(TARGETS ${src_lib_shared_nupiccore} + DESTINATION bin) install(DIRECTORY nupic/ DESTINATION include/nupic From fb87ba80758b0ade9123c8331fc5e53e3a1a3cc7 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 15:34:55 -0800 Subject: [PATCH 06/44] NUP-2320 Include necessary py_support files in nupic_core shared lib (needed by PyRegion). Added temp `ls` statement to help debug why libnupic_core.so isn't being found in most Bamboo build targets. --- ci/build-and-test-nupic-bindings.sh | 1 + src/CMakeLists.txt | 76 +++++++++++++---------------- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/ci/build-and-test-nupic-bindings.sh b/ci/build-and-test-nupic-bindings.sh index fe4bbfc6b0..69114f41d1 100755 --- a/ci/build-and-test-nupic-bindings.sh +++ b/ci/build-and-test-nupic-bindings.sh @@ -141,6 +141,7 @@ pip install ${PIP_USER} \ # Run the nupic.core c++ tests cd ${NUPIC_CORE_ROOT}/build/release/bin +ls # ZZZ checking for presence of libnupic_core.so to debug no such file or directory error on Bamboo build ./cpp_region_test ./py_region_test ./unit_tests diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb71222592..5ff0eace10 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -197,22 +197,19 @@ message(STATUS "PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") # -# Python support +# Source files that support Python-specific logic in nupic_core shared lib (e.g., +# PyRegion) # -set(src_py_support_files +set(src_core_py_support_sources nupic/py_support/NumpyVector.cpp nupic/py_support/PyArray.cpp nupic/py_support/PyHelpers.cpp - nupic/py_support/PythonStream.cpp nupic/py_support/PyCapnp.cpp - nupic/bindings/PySparseTensor.cpp ) -# # pycapnp integration -# if(NOT "${PLATFORM}" STREQUAL "windows") - list(APPEND src_py_support_files nupic/py_support/CapnpToPycapnp.cpp) + list(APPEND src_core_py_support_sources nupic/py_support/CapnpToPycapnp.cpp) # Add the pycapnp Python package headers to includes execute_process( @@ -236,6 +233,7 @@ if(NOT "${PLATFORM}" STREQUAL "windows") endif() + # List all .capnp files here. The C++ files will be generated and included # when compiling later on. set(src_capnp_specs_rel @@ -387,14 +385,15 @@ set(src_nupiccore_internals_compile_flags message(STATUS "src_compile_flags = ${src_compile_flags}") message(STATUS "src_nupiccore_internals_compile_flags = ${src_nupiccore_internals_compile_flags}") + # # Create the nupic_core shared library containing nupic core internals (sans -# python support) and our externals +# python binding support) as well as our externals # set(src_lib_shared_nupiccore nupic_core) -set(src_external_static_libs +set(src_lib_shared_nupiccore_link_libraries ${YAML_CPP_STATIC_LIB_TARGET} ${YAML_STATIC_LIB_TARGET} ${APR1_STATIC_LIB_TARGET} @@ -405,10 +404,9 @@ if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run without our python # extensions, add capnproto static lib. Our python extensions have their own # logic governing whether to include capnproto. - list(APPEND src_external_static_libs ${CAPNP_STATIC_LIB_TARGET}) + list(APPEND src_lib_shared_nupiccore_link_libraries ${CAPNP_STATIC_LIB_TARGET}) endif() -set(src_lib_shared_nupiccore_link_libraries ${src_external_static_libs}) if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run outside the python # environment, add dependency on python lib to resolve python symbol @@ -416,7 +414,9 @@ if (NOT NUPIC_BUILD_PYEXT_MODULES) list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) endif() -add_library(${src_lib_shared_nupiccore} SHARED ${src_nupiccore_internal_srcs}) +add_library(${src_lib_shared_nupiccore} SHARED + ${src_nupiccore_internal_srcs} + ${src_core_py_support_sources}) target_link_libraries(${src_lib_shared_nupiccore} ${src_lib_shared_nupiccore_link_libraries}) @@ -457,31 +457,29 @@ endif() # -# Create py_support "object" library that generates python support object files -# for linking into python extensions and c++ tests that need it. +# Archive that supports Python extensions (also included in our C++ tests) # -set(src_lib_py_support_objects py_support_objects) +set(src_py_binding_support_sources + nupic/py_support/PythonStream.cpp + nupic/bindings/PySparseTensor.cpp) -# We use this generator value to add the python support objects to the -# source lists of the modules that rely on them. -set(src_py_support_object_list_gen - "$") +set(src_lib_static_py_binding_support py_binding_support_archive) -add_library(${src_lib_py_support_objects} OBJECT - ${src_py_support_files}) +add_library(${src_lib_static_py_binding_support} STATIC + ${src_py_binding_support_sources}) # We rely on artifacts from building of nupic_core shared lib (e.g., # capnp schema C++ headers) -add_dependencies(${src_lib_py_support_objects} - ${src_lib_shared_nupiccore}) +#add_dependencies(${src_lib_static_py_binding_support} +# ${src_lib_shared_nupiccore}) -set_target_properties(${src_lib_py_support_objects} PROPERTIES COMPILE_FLAGS +set_target_properties(${src_lib_static_py_binding_support} PROPERTIES COMPILE_FLAGS ${src_nupiccore_internals_compile_flags}) if(${NUPIC_IWYU}) # TODO: Create a target that doesn't include the generated capnp schema files # since we don't want to run iwyu on them and iwyu can only be applied to # CMake targets (not source files directly). - set_target_properties(${src_lib_py_support_objects} + set_target_properties(${src_lib_static_py_binding_support} PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() @@ -495,7 +493,8 @@ endif() set(src_common_test_exe_libs ${src_lib_shared_nupiccore} ${PYTHON_LIBRARIES} - ${src_common_os_libs}) + ${src_common_os_libs} + ${src_lib_static_py_binding_support}) # Add capnproto static lib to our C/C++ test apps when building python # extensions, since capnproto objects are excluded from nupic_core combined lib @@ -512,8 +511,7 @@ message(STATUS "src_common_test_exe_libs = ${src_common_test_exe_libs}") # set(src_executable_cppregiontest cpp_region_test) add_executable(${src_executable_cppregiontest} - test/integration/CppRegionTest.cpp - ${src_py_support_object_list_gen}) + test/integration/CppRegionTest.cpp) target_link_libraries(${src_executable_cppregiontest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_cppregiontest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_cppregiontest} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") @@ -528,8 +526,7 @@ add_custom_target(tests_cpp_region # set(src_executable_pyregiontest py_region_test) add_executable(${src_executable_pyregiontest} - test/integration/PyRegionTest.cpp - ${src_py_support_object_list_gen}) + test/integration/PyRegionTest.cpp) target_link_libraries(${src_executable_pyregiontest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_pyregiontest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -546,8 +543,7 @@ add_custom_target(tests_py_region # set(src_executable_connectionsperformancetest connections_performance_test) add_executable(${src_executable_connectionsperformancetest} - test/integration/ConnectionsPerformanceTest.cpp - ${src_py_support_object_list_gen}) + test/integration/ConnectionsPerformanceTest.cpp) target_link_libraries(${src_executable_connectionsperformancetest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_connectionsperformancetest} @@ -565,8 +561,7 @@ add_custom_target(tests_connections_performance # set(src_executable_helloregion helloregion) add_executable(${src_executable_helloregion} - examples/regions/HelloRegions.cpp - ${src_py_support_object_list_gen}) + examples/regions/HelloRegions.cpp) target_link_libraries(${src_executable_helloregion} ${src_common_test_exe_libs}) set_target_properties(${src_executable_helloregion} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -578,8 +573,7 @@ set_target_properties(${src_executable_helloregion} # set(src_executable_prototest prototest) add_executable(${src_executable_prototest} - examples/prototest.cpp - ${src_py_support_object_list_gen}) + examples/prototest.cpp) target_link_libraries(${src_executable_prototest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_prototest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -591,8 +585,7 @@ set_target_properties(${src_executable_prototest} # set(src_executable_hellosptp hello_sp_tp) add_executable(${src_executable_hellosptp} - examples/algorithms/HelloSP_TP.cpp - ${src_py_support_object_list_gen}) + examples/algorithms/HelloSP_TP.cpp) target_link_libraries(${src_executable_hellosptp} ${src_common_test_exe_libs}) set_target_properties(${src_executable_hellosptp} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) @@ -605,7 +598,6 @@ set_target_properties(${src_executable_hellosptp} # set(src_executable_gtests unit_tests) add_executable(${src_executable_gtests} - ${src_py_support_object_list_gen} test/unit/algorithms/AnomalyTest.cpp test/unit/algorithms/CondProbTableTest.cpp test/unit/algorithms/ConnectionsTest.cpp @@ -748,7 +740,8 @@ if (NUPIC_BUILD_PYEXT_MODULES) # in our own compilation of capnproto on objects created by pycapnp's. set(_SRC_SWIG_LINK_LIBRARIES ${src_lib_shared_nupiccore} - ${src_common_os_libs}) + ${src_common_os_libs} + ${src_lib_static_py_binding_support}) # Common dependencies for our python extensions for use with # SWIG_MODULE_name_EXTRA_DEPS @@ -819,8 +812,7 @@ if (NUPIC_BUILD_PYEXT_MODULES) # Note: swig_add_module outputs ${swig_generated_file_fullname} swig_add_module(${MODULE_NAME} python - ${source_interface_file} - ${src_py_support_object_list_gen}) + ${source_interface_file}) set_source_files_properties( ${swig_generated_file_fullname} PROPERTIES From 343e6e11a7c89e55396a1b8d042a49fd5b70d673 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 16:46:46 -0800 Subject: [PATCH 07/44] NUP-2320 Add missing link libraries to nupic_core shared lib build on Windows. --- src/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ff0eace10..21aab03c06 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -412,8 +412,20 @@ if (NOT NUPIC_BUILD_PYEXT_MODULES) # environment, add dependency on python lib to resolve python symbol # references (e.g., from PyRegion) list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) +elseif("${PLATFORM}" STREQUAL "windows") + # NOTE Windows DLLs are shared executables with their own main; they require + # all symbols to resolve at link time, so we have to add libpython for this + # platform + # + # NOTE On Windows builds, we include capnproto because we + # presently build self-contained CAPNP_LITE on Windows, and Windows + # nupic/nupic.bindings presently have no dependency on pycapnp. + list(APPEND src_lib_shared_nupiccore_link_libraries + ${PYTHON_LIBRARIES} + ${CAPNP_STATIC_LIB_TARGET}) endif() + add_library(${src_lib_shared_nupiccore} SHARED ${src_nupiccore_internal_srcs} ${src_core_py_support_sources}) From 0ed564282bc58b9a6fd8d8aa012ddb51c7b453bb Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 17:06:54 -0800 Subject: [PATCH 08/44] NUP-2320 - Link nupic_core shared lib against necessary os libs. --- src/CMakeLists.txt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 21aab03c06..f179c00bd8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -393,6 +393,17 @@ message(STATUS "src_nupiccore_internals_compile_flags = ${src_nupiccore_internal set(src_lib_shared_nupiccore nupic_core) +# NOTE Non-Windows shared lib shouldn't be linking agains libpython; +# symbols should be available automatically when python loads the extension. +# +# NOTE We don't link our non-Windows shared lib with capnproto static +# lib in order to force runtime linkage to pycapnp's capnproto symbols thus +# avoiding the conflict of executing the methods in our own +# compilation of capnproto (different compiler/STL/version, flags, etc.) on +# objects created by pycapnp's. We force runtime linking to pycapnp's +# capnproto symbols by preloading the pycapnp extension in our extension +# python proxy modules, thus avoiding the conflict from executing the methods +# in our own compilation of capnproto on objects created by pycapnp's. set(src_lib_shared_nupiccore_link_libraries ${YAML_CPP_STATIC_LIB_TARGET} ${YAML_STATIC_LIB_TARGET} @@ -425,6 +436,8 @@ elseif("${PLATFORM}" STREQUAL "windows") ${CAPNP_STATIC_LIB_TARGET}) endif() +list(APPEND src_lib_shared_nupiccore_link_libraries ${src_common_os_libs}) + add_library(${src_lib_shared_nupiccore} SHARED ${src_nupiccore_internal_srcs} @@ -751,9 +764,9 @@ if (NUPIC_BUILD_PYEXT_MODULES) # python proxy modules, thus avoiding the conflict from executing the methods # in our own compilation of capnproto on objects created by pycapnp's. set(_SRC_SWIG_LINK_LIBRARIES + ${src_lib_static_py_binding_support} ${src_lib_shared_nupiccore} - ${src_common_os_libs} - ${src_lib_static_py_binding_support}) + ${src_common_os_libs}) # Common dependencies for our python extensions for use with # SWIG_MODULE_name_EXTRA_DEPS From b6d74a5a4f41a6e2e559a73ce3096df1f2949ee8 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 17:17:33 -0800 Subject: [PATCH 09/44] NUP-2320 Make py_binding_support_archive build depend on nupic_core shared lib to force generation of capnp schema. --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f179c00bd8..2e508461b1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -495,8 +495,8 @@ add_library(${src_lib_static_py_binding_support} STATIC # We rely on artifacts from building of nupic_core shared lib (e.g., # capnp schema C++ headers) -#add_dependencies(${src_lib_static_py_binding_support} -# ${src_lib_shared_nupiccore}) +add_dependencies(${src_lib_static_py_binding_support} + ${src_lib_shared_nupiccore}) set_target_properties(${src_lib_static_py_binding_support} PROPERTIES COMPILE_FLAGS ${src_nupiccore_internals_compile_flags}) From 221a9290f2c1ce0a025dfe17738276682a17f492 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 25 Jan 2017 18:47:19 -0800 Subject: [PATCH 10/44] NUP-2320 Force visibility of symbols from internal sources compiled into nupic_core shared lib that don't have explicit visibility to avoid prining when building on Windows. --- CommonCompilerConfig.cmake | 11 +++++++++++ src/CMakeLists.txt | 9 ++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index 783d019e61..eee48d33f4 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -62,6 +62,10 @@ # # INTERNAL_CXX_FLAGS_OPTIMIZED: string of C++ flags with explicit optimization flags for internal sources # +# INTERNAL_CXX_FLAGS_NO_HIDE: string of C++ flags that may be appended to +# INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols +# that have default visibility. +# # INTERNAL_LINKER_FLAGS_OPTIMIZED: string of linker flags for linking internal executables # and shared libraries (DLLs) with optimizations that are # compatible with INTERNAL_CXX_FLAGS_OPTIMIZED @@ -97,6 +101,7 @@ set(COMMON_COMPILER_DEFINITIONS) set(COMMON_COMPILER_DEFINITIONS_STR) set(INTERNAL_CXX_FLAGS_OPTIMIZED) +set(INTERNAL_CXX_FLAGS_NO_HIDE) set(INTERNAL_LINKER_FLAGS_OPTIMIZED) set(PYEXT_CXX_FLAGS_OPTIMIZED) @@ -243,6 +248,7 @@ set(fail_link_on_undefined_symbols_flags "") set(allow_link_with_undefined_symbols_flags "") set(cxx_hidden_vis_compile_flags "") set(shared_hidden_vis_compile_flags "") +set(shared_default_to_vis_compile_flags "") if(${CMAKE_CXX_COMPILER_ID} STREQUAL "MSVC") # MS Visual C @@ -263,6 +269,10 @@ else() set(cxx_hidden_vis_compile_flags "${cxx_hidden_vis_compile_flags} -fvisibility-inlines-hidden") set(shared_hidden_vis_compile_flags "${shared_hidden_vis_compile_flags} -fvisibility=hidden") + # Per https://gcc.gnu.org/onlinedocs/gcc-4.1.2/gcc/Code-Gen-Options.html: + # "Despite the nomenclature, default always means public" + set(shared_default_to_vis_compile_flags "${shared_default_to_vis_compile_flags} -fvisibility=default") + set(shared_compile_flags "${shared_compile_flags} ${stdlib_common} -fdiagnostics-show-option") set (internal_compiler_warning_flags "${internal_compiler_warning_flags} -Werror -Wextra -Wreturn-type -Wunused -Wno-unused-variable -Wno-unused-parameter -Wno-missing-field-initializers") set (external_compiler_warning_flags "${external_compiler_warning_flags} -Wno-unused-variable -Wno-unused-parameter -Wno-incompatible-pointer-types -Wno-deprecated-declarations") @@ -363,6 +373,7 @@ endif() # Settings for internal nupic.core code set(INTERNAL_CXX_FLAGS_OPTIMIZED "${build_type_specific_compile_flags} ${shared_compile_flags} ${cxx_flags_unoptimized} ${internal_compiler_warning_flags} ${optimization_flags_cc}") +set(INTERNAL_CXX_FLAGS_NO_HIDE "${shared_default_to_vis_compile_flags}") set(complete_linker_flags_unoptimized "${build_type_specific_linker_flags} ${shared_linker_flags_unoptimized}") set(complete_linker_flags_unoptimized "${complete_linker_flags_unoptimized} ${fail_link_on_undefined_symbols_flags}") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2e508461b1..8f1d15eb4a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -446,8 +446,11 @@ add_library(${src_lib_shared_nupiccore} SHARED target_link_libraries(${src_lib_shared_nupiccore} ${src_lib_shared_nupiccore_link_libraries}) +# NOTE We use INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols that +# have default visibility. This is necessary for the Windows build, where +# symbols default to hidden visibility. set_target_properties(${src_lib_shared_nupiccore} PROPERTIES COMPILE_FLAGS - ${src_nupiccore_internals_compile_flags}) + "${src_nupiccore_internals_compile_flags} ${INTERNAL_CXX_FLAGS_NO_HIDE}") if(${NUPIC_IWYU}) # TODO: Create a target that doesn't include the generated capnp schema files # since we don't want to run iwyu on them and iwyu can only be applied to @@ -516,10 +519,10 @@ endif() # Common libs for test executables set(src_common_test_exe_libs + ${src_lib_static_py_binding_support} ${src_lib_shared_nupiccore} ${PYTHON_LIBRARIES} - ${src_common_os_libs} - ${src_lib_static_py_binding_support}) + ${src_common_os_libs}) # Add capnproto static lib to our C/C++ test apps when building python # extensions, since capnproto objects are excluded from nupic_core combined lib From 38b32c21c95724403301553570d141a913472e90 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Thu, 26 Jan 2017 23:42:30 -0800 Subject: [PATCH 11/44] NUP-2320 Use rpath on Unix (OS X and Linux) systems to find nupic_core shared lib. --- src/CMakeLists.txt | 70 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8f1d15eb4a..06b4c0a1ed 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -393,6 +393,19 @@ message(STATUS "src_nupiccore_internals_compile_flags = ${src_nupiccore_internal set(src_lib_shared_nupiccore nupic_core) +# Define install_rpath for Unix systems (e.g., Apple and Linux); NOTE Windows +# doesn't have the equivalent of rpath. +if(UNIX) + # Set INSTALL_RPATH to the relocatable location where nupic_core shared + # lib may be found + if(APPLE) + set(_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH "@loader_path") + else() + # non-apple Unix, such as Linux + set(_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH "\$ORIGIN") + endif() +endif() + # NOTE Non-Windows shared lib shouldn't be linking agains libpython; # symbols should be available automatically when python loads the extension. # @@ -474,12 +487,21 @@ else() endif() if (APPLE) - set_target_properties(${src_lib_shared_nupiccore} PROPERTIES MACOSX_RPATH OFF) + # Enable MACOSX_RPATH to use @rpath as install_name of this shared lib to + # allow client binary (app or lib) to locate it via the RPATH mechanism. + # This is specific to OS X; linux simply requires setting of RPATH in the + # client binary. NOTE The RPATH mechanism facilitates relocatable installation + # of packages. + set_target_properties(${src_lib_shared_nupiccore} PROPERTIES MACOSX_RPATH ON) endif() if (NUPIC_BUILD_PYEXT_MODULES AND PY_EXTENSIONS_DIR) - set_target_properties(${src_lib_shared_nupiccore} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${PY_EXTENSIONS_DIR}) + # Request installtion into nupic.bindings's tree. We install it into the same + # directory as our extensions in order to facilitate relocatable loading on + # Windows, which doesn't support the RPATH mechanism. + install(TARGETS ${src_lib_shared_nupiccore} + DESTINATION ${PY_EXTENSIONS_DIR}) + endif() @@ -543,6 +565,11 @@ add_executable(${src_executable_cppregiontest} target_link_libraries(${src_executable_cppregiontest} ${src_common_test_exe_libs}) set_target_properties(${src_executable_cppregiontest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_cppregiontest} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_cppregiontest} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() add_custom_target(tests_cpp_region COMMAND ${src_executable_cppregiontest} DEPENDS ${src_executable_cppregiontest} @@ -560,6 +587,11 @@ set_target_properties(${src_executable_pyregiontest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_pyregiontest} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_pyregiontest} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() add_custom_target(tests_py_region COMMAND ${src_executable_pyregiontest} DEPENDS ${src_executable_pyregiontest} @@ -578,6 +610,11 @@ set_target_properties(${src_executable_connectionsperformancetest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_connectionsperformancetest} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_connectionsperformancetest} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() add_custom_target(tests_connections_performance COMMAND ${src_executable_connectionsperformancetest} DEPENDS ${src_executable_connectionsperformancetest} @@ -595,6 +632,11 @@ set_target_properties(${src_executable_helloregion} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_helloregion} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_helloregion} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() # # Setup prototest example @@ -607,6 +649,11 @@ set_target_properties(${src_executable_prototest} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_prototest} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_prototest} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() # # Setup HelloSP_TP example @@ -619,6 +666,11 @@ set_target_properties(${src_executable_hellosptp} PROPERTIES COMPILE_FLAGS ${src_compile_flags}) set_target_properties(${src_executable_hellosptp} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_hellosptp} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() # @@ -681,6 +733,11 @@ target_link_libraries(${src_executable_gtests} set_target_properties(${src_executable_gtests} PROPERTIES COMPILE_FLAGS ${src_compile_flags} LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") +if(UNIX) + set_target_properties( + ${src_executable_gtests} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") +endif() add_custom_target(tests_unit COMMAND ${src_executable_gtests} DEPENDS ${src_executable_gtests} @@ -889,6 +946,13 @@ if (NUPIC_BUILD_PYEXT_MODULES) # If a path is specified, copy extensions files to proper location. if (PY_EXTENSIONS_DIR) + if(UNIX) + # NOTE Windows doesn't have the equivalent of rpath. + set_target_properties( + ${real_target} PROPERTIES + INSTALL_RPATH "${_SRC_LIB_SHARED_NUPICCORE_INSTALL_RPATH}") + endif() + install(TARGETS ${real_target} LIBRARY DESTINATION ${PY_EXTENSIONS_DIR}) From 152f5d68f0dd31b9c7d31d2b4547d1ab09ba49dc Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 27 Jan 2017 01:08:25 -0800 Subject: [PATCH 12/44] NUP-2320 When generating python extensions, link tests also with a shared library containing capnproto to satisfy capnp references that would normally be satisfied by capnproto in pycapnp. --- src/CMakeLists.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 06b4c0a1ed..38fccf10c5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -546,11 +546,22 @@ set(src_common_test_exe_libs ${PYTHON_LIBRARIES} ${src_common_os_libs}) -# Add capnproto static lib to our C/C++ test apps when building python +# Add capnproto lib to our C/C++ test apps when building python # extensions, since capnproto objects are excluded from nupic_core combined lib # to avoid conflict with capnproto methods compiled into the pycapnp extension if (NUPIC_BUILD_PYEXT_MODULES) - list(APPEND src_common_test_exe_libs ${CAPNP_STATIC_LIB_TARGET}) + add_library("capnprotoshared" SHARED) + target_link_libraries("capnprotoshared" ${CAPNP_STATIC_LIB_TARGET}) + set_target_properties("capnprotoshared" PROPERTIES LINK_FLAGS + "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") + if (APPLE) + set_target_properties("capnprotoshared" PROPERTIES MACOSX_RPATH ON) + endif() + + install(TARGETS "capnprotoshared" + DESTINATION bin) + + list(APPEND src_common_test_exe_libs "capnprotoshared") endif() message(STATUS "src_common_test_exe_libs = ${src_common_test_exe_libs}") From 9fed2c7cf5b4ed32af28c2eb1a9b54ecdc1ede23 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 27 Jan 2017 16:24:57 -0800 Subject: [PATCH 13/44] NUP-2320 Use toolchain-specific linker option to force all archive symbols to be added to shared libs. --- src/CMakeLists.txt | 64 +++++++++++++++++++++++++++++++--------------- src/empty.cpp | 8 ++++++ 2 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 src/empty.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 38fccf10c5..d14f673cfa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -55,6 +55,19 @@ if(${NUPIC_IWYU}) endif() +# +# Linker options +# +if(MSVC) + set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "/WHOLEARCHIVE") +elseif(APPLE) + set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "-all_load") +else() + # Linux and MINGW + set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "--whole-archive") +endif() + + # # Set up compile flags for internal sources and for swig-generated sources # @@ -418,24 +431,24 @@ endif() # python proxy modules, thus avoiding the conflict from executing the methods # in our own compilation of capnproto on objects created by pycapnp's. set(src_lib_shared_nupiccore_link_libraries - ${YAML_CPP_STATIC_LIB_TARGET} - ${YAML_STATIC_LIB_TARGET} - ${APR1_STATIC_LIB_TARGET} - ${APRUTIL1_STATIC_LIB_TARGET} - ${Z_STATIC_LIB_TARGET}) + PRIVATE ${YAML_CPP_STATIC_LIB_TARGET} + PRIVATE ${YAML_STATIC_LIB_TARGET} + PRIVATE ${APR1_STATIC_LIB_TARGET} + PRIVATE ${APRUTIL1_STATIC_LIB_TARGET} + PRIVATE ${Z_STATIC_LIB_TARGET}) if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run without our python # extensions, add capnproto static lib. Our python extensions have their own # logic governing whether to include capnproto. - list(APPEND src_lib_shared_nupiccore_link_libraries ${CAPNP_STATIC_LIB_TARGET}) + list(APPEND src_lib_shared_nupiccore_link_libraries "PRIVATE ${CAPNP_STATIC_LIB_TARGET}") endif() if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run outside the python # environment, add dependency on python lib to resolve python symbol # references (e.g., from PyRegion) - list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) + list(APPEND src_lib_shared_nupiccore_link_libraries "PRIVATE ${PYTHON_LIBRARIES}") elseif("${PLATFORM}" STREQUAL "windows") # NOTE Windows DLLs are shared executables with their own main; they require # all symbols to resolve at link time, so we have to add libpython for this @@ -445,13 +458,12 @@ elseif("${PLATFORM}" STREQUAL "windows") # presently build self-contained CAPNP_LITE on Windows, and Windows # nupic/nupic.bindings presently have no dependency on pycapnp. list(APPEND src_lib_shared_nupiccore_link_libraries - ${PYTHON_LIBRARIES} - ${CAPNP_STATIC_LIB_TARGET}) + "PRIVATE ${PYTHON_LIBRARIES}" + "PRIVATE ${CAPNP_STATIC_LIB_TARGET}") endif() list(APPEND src_lib_shared_nupiccore_link_libraries ${src_common_os_libs}) - add_library(${src_lib_shared_nupiccore} SHARED ${src_nupiccore_internal_srcs} ${src_core_py_support_sources}) @@ -478,12 +490,12 @@ if (NUPIC_BUILD_PYEXT_MODULES) # that will be provided by the python runtime as well as capnproto symbols # that will be provided by pycapnp. set_target_properties(${src_lib_shared_nupiccore} PROPERTIES LINK_FLAGS - "${PYEXT_LINKER_FLAGS_OPTIMIZED}") + "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") else() # When building a standalone shared library, specifically don't allow link # with undefined symbols set_target_properties(${src_lib_shared_nupiccore} PROPERTIES LINK_FLAGS - "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") + "${INTERNAL_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") endif() if (APPLE) @@ -547,21 +559,31 @@ set(src_common_test_exe_libs ${src_common_os_libs}) # Add capnproto lib to our C/C++ test apps when building python -# extensions, since capnproto objects are excluded from nupic_core combined lib -# to avoid conflict with capnproto methods compiled into the pycapnp extension +# extensions, since capnproto objects are excluded from nupic_core shared lib +# in this case to avoid conflict with capnproto methods compiled into the +# pycapnp extension if (NUPIC_BUILD_PYEXT_MODULES) - add_library("capnprotoshared" SHARED) - target_link_libraries("capnprotoshared" ${CAPNP_STATIC_LIB_TARGET}) - set_target_properties("capnprotoshared" PROPERTIES LINK_FLAGS - "${INTERNAL_LINKER_FLAGS_OPTIMIZED}") + set(_SRC_LIB_SHARED_CAPNPROTO capnproto_shared) + add_library(${_SRC_LIB_SHARED_CAPNPROTO} SHARED "empty.cpp") + set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES COMPILE_FLAGS + "${src_nupiccore_internals_compile_flags}") + + target_link_libraries(${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE ${CAPNP_STATIC_LIB_TARGET}) + + # Make sure the entire capnproto archive is added to the shared lib + set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES LINK_FLAGS + "${INTERNAL_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") + if (APPLE) - set_target_properties("capnprotoshared" PROPERTIES MACOSX_RPATH ON) + set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES MACOSX_RPATH ON) endif() - install(TARGETS "capnprotoshared" + install(TARGETS ${_SRC_LIB_SHARED_CAPNPROTO} DESTINATION bin) - list(APPEND src_common_test_exe_libs "capnprotoshared") + set(src_common_test_exe_libs + ${_SRC_LIB_SHARED_CAPNPROTO} + ${src_common_test_exe_libs}) endif() message(STATUS "src_common_test_exe_libs = ${src_common_test_exe_libs}") diff --git a/src/empty.cpp b/src/empty.cpp new file mode 100644 index 0000000000..70250c011f --- /dev/null +++ b/src/empty.cpp @@ -0,0 +1,8 @@ +// Needed for cmake add_library, where all code is in archives. + +extern int myverylongfuncthattakesvoid(void); +int myverylongfuncthattakesvoid(void) +{ + return 0; + +} From 0bd12fb3be5a01e8d66b57919b4409cb3f7c6ced Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 27 Jan 2017 16:48:12 -0800 Subject: [PATCH 14/44] NUP-2320 Make all shared lib target_link_library items PRIVATE. --- ci/build-and-test-nupic-bindings.sh | 1 - src/CMakeLists.txt | 30 ++++++++++++++--------------- src/empty.cpp | 6 ------ 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/ci/build-and-test-nupic-bindings.sh b/ci/build-and-test-nupic-bindings.sh index 69114f41d1..fe4bbfc6b0 100755 --- a/ci/build-and-test-nupic-bindings.sh +++ b/ci/build-and-test-nupic-bindings.sh @@ -141,7 +141,6 @@ pip install ${PIP_USER} \ # Run the nupic.core c++ tests cd ${NUPIC_CORE_ROOT}/build/release/bin -ls # ZZZ checking for presence of libnupic_core.so to debug no such file or directory error on Bamboo build ./cpp_region_test ./py_region_test ./unit_tests diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d14f673cfa..6edeb1ac4a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -213,7 +213,7 @@ message(STATUS "PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") # Source files that support Python-specific logic in nupic_core shared lib (e.g., # PyRegion) # -set(src_core_py_support_sources +set(src_core_py_support_srcs nupic/py_support/NumpyVector.cpp nupic/py_support/PyArray.cpp nupic/py_support/PyHelpers.cpp @@ -222,7 +222,7 @@ set(src_core_py_support_sources # pycapnp integration if(NOT "${PLATFORM}" STREQUAL "windows") - list(APPEND src_core_py_support_sources nupic/py_support/CapnpToPycapnp.cpp) + list(APPEND src_core_py_support_srcs nupic/py_support/CapnpToPycapnp.cpp) # Add the pycapnp Python package headers to includes execute_process( @@ -431,24 +431,24 @@ endif() # python proxy modules, thus avoiding the conflict from executing the methods # in our own compilation of capnproto on objects created by pycapnp's. set(src_lib_shared_nupiccore_link_libraries - PRIVATE ${YAML_CPP_STATIC_LIB_TARGET} - PRIVATE ${YAML_STATIC_LIB_TARGET} - PRIVATE ${APR1_STATIC_LIB_TARGET} - PRIVATE ${APRUTIL1_STATIC_LIB_TARGET} - PRIVATE ${Z_STATIC_LIB_TARGET}) + ${YAML_CPP_STATIC_LIB_TARGET} + ${YAML_STATIC_LIB_TARGET} + ${APR1_STATIC_LIB_TARGET} + ${APRUTIL1_STATIC_LIB_TARGET} + ${Z_STATIC_LIB_TARGET}) if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run without our python # extensions, add capnproto static lib. Our python extensions have their own # logic governing whether to include capnproto. - list(APPEND src_lib_shared_nupiccore_link_libraries "PRIVATE ${CAPNP_STATIC_LIB_TARGET}") + list(APPEND src_lib_shared_nupiccore_link_libraries ${CAPNP_STATIC_LIB_TARGET}) endif() if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run outside the python # environment, add dependency on python lib to resolve python symbol # references (e.g., from PyRegion) - list(APPEND src_lib_shared_nupiccore_link_libraries "PRIVATE ${PYTHON_LIBRARIES}") + list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) elseif("${PLATFORM}" STREQUAL "windows") # NOTE Windows DLLs are shared executables with their own main; they require # all symbols to resolve at link time, so we have to add libpython for this @@ -458,18 +458,18 @@ elseif("${PLATFORM}" STREQUAL "windows") # presently build self-contained CAPNP_LITE on Windows, and Windows # nupic/nupic.bindings presently have no dependency on pycapnp. list(APPEND src_lib_shared_nupiccore_link_libraries - "PRIVATE ${PYTHON_LIBRARIES}" - "PRIVATE ${CAPNP_STATIC_LIB_TARGET}") + ${PYTHON_LIBRARIES} + ${CAPNP_STATIC_LIB_TARGET}) endif() list(APPEND src_lib_shared_nupiccore_link_libraries ${src_common_os_libs}) add_library(${src_lib_shared_nupiccore} SHARED ${src_nupiccore_internal_srcs} - ${src_core_py_support_sources}) + ${src_core_py_support_srcs}) target_link_libraries(${src_lib_shared_nupiccore} - ${src_lib_shared_nupiccore_link_libraries}) + PRIVATE ${src_lib_shared_nupiccore_link_libraries}) # NOTE We use INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols that # have default visibility. This is necessary for the Windows build, where @@ -521,14 +521,14 @@ endif() # # Archive that supports Python extensions (also included in our C++ tests) # -set(src_py_binding_support_sources +set(src_py_binding_support_srcs nupic/py_support/PythonStream.cpp nupic/bindings/PySparseTensor.cpp) set(src_lib_static_py_binding_support py_binding_support_archive) add_library(${src_lib_static_py_binding_support} STATIC - ${src_py_binding_support_sources}) + ${src_py_binding_support_srcs}) # We rely on artifacts from building of nupic_core shared lib (e.g., # capnp schema C++ headers) diff --git a/src/empty.cpp b/src/empty.cpp index 70250c011f..7fe2f8a987 100644 --- a/src/empty.cpp +++ b/src/empty.cpp @@ -1,8 +1,2 @@ // Needed for cmake add_library, where all code is in archives. -extern int myverylongfuncthattakesvoid(void); -int myverylongfuncthattakesvoid(void) -{ - return 0; - -} From 94aea65b4d4b2dc6c1b36b843a6552971a3aac41 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 27 Jan 2017 17:07:17 -0800 Subject: [PATCH 15/44] NUP-2320 Use `-Wl,--whole-archive` instead of raw `--whole-archive` for Linux and mingw (was failing on mingw without -Wl) --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6edeb1ac4a..3d821b0e0d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,7 +64,7 @@ elseif(APPLE) set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "-all_load") else() # Linux and MINGW - set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "--whole-archive") + set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "-Wl,--whole-archive") endif() From 47b561d392ce273a35e6f18d80b2661e62b45478 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 27 Jan 2017 17:28:10 -0800 Subject: [PATCH 16/44] Convert new private cmake vars to upper case prefixed with underscore --- src/CMakeLists.txt | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3d821b0e0d..0f3637b54f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -213,7 +213,7 @@ message(STATUS "PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") # Source files that support Python-specific logic in nupic_core shared lib (e.g., # PyRegion) # -set(src_core_py_support_srcs +set(_SRC_CORE_PY_SUPPORT_SRCS nupic/py_support/NumpyVector.cpp nupic/py_support/PyArray.cpp nupic/py_support/PyHelpers.cpp @@ -222,7 +222,7 @@ set(src_core_py_support_srcs # pycapnp integration if(NOT "${PLATFORM}" STREQUAL "windows") - list(APPEND src_core_py_support_srcs nupic/py_support/CapnpToPycapnp.cpp) + list(APPEND _SRC_CORE_PY_SUPPORT_SRCS nupic/py_support/CapnpToPycapnp.cpp) # Add the pycapnp Python package headers to includes execute_process( @@ -315,7 +315,7 @@ message(STATUS "src_common_os_libs = ${src_common_os_libs}") # # Setup a list of our own sources without py support # -set(src_nupiccore_internal_srcs +set(_SRC_NUPICCORE_INTERNAL_SRCS nupic/algorithms/Anomaly.cpp nupic/algorithms/BitHistory.cpp nupic/algorithms/Cell.cpp @@ -390,7 +390,7 @@ set(src_nupiccore_internal_srcs nupic/utils/TRandom.cpp nupic/utils/Watcher.cpp) -list(APPEND src_nupiccore_internal_srcs ${src_capnp_generated_srcs}) +list(APPEND _SRC_NUPICCORE_INTERNAL_SRCS ${src_capnp_generated_srcs}) set(src_nupiccore_internals_compile_flags "${src_compile_flags} -I${src_numpy_core}/include") @@ -404,7 +404,7 @@ message(STATUS "src_nupiccore_internals_compile_flags = ${src_nupiccore_internal # python binding support) as well as our externals # -set(src_lib_shared_nupiccore nupic_core) +set(_SRC_LIB_SHARED_NUPICCORE nupic_core) # Define install_rpath for Unix systems (e.g., Apple and Linux); NOTE Windows # doesn't have the equivalent of rpath. @@ -430,7 +430,7 @@ endif() # capnproto symbols by preloading the pycapnp extension in our extension # python proxy modules, thus avoiding the conflict from executing the methods # in our own compilation of capnproto on objects created by pycapnp's. -set(src_lib_shared_nupiccore_link_libraries +set(_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${YAML_CPP_STATIC_LIB_TARGET} ${YAML_STATIC_LIB_TARGET} ${APR1_STATIC_LIB_TARGET} @@ -441,14 +441,14 @@ if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run without our python # extensions, add capnproto static lib. Our python extensions have their own # logic governing whether to include capnproto. - list(APPEND src_lib_shared_nupiccore_link_libraries ${CAPNP_STATIC_LIB_TARGET}) + list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${CAPNP_STATIC_LIB_TARGET}) endif() if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run outside the python # environment, add dependency on python lib to resolve python symbol # references (e.g., from PyRegion) - list(APPEND src_lib_shared_nupiccore_link_libraries ${PYTHON_LIBRARIES}) + list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES}) elseif("${PLATFORM}" STREQUAL "windows") # NOTE Windows DLLs are shared executables with their own main; they require # all symbols to resolve at link time, so we have to add libpython for this @@ -457,30 +457,30 @@ elseif("${PLATFORM}" STREQUAL "windows") # NOTE On Windows builds, we include capnproto because we # presently build self-contained CAPNP_LITE on Windows, and Windows # nupic/nupic.bindings presently have no dependency on pycapnp. - list(APPEND src_lib_shared_nupiccore_link_libraries + list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES} ${CAPNP_STATIC_LIB_TARGET}) endif() -list(APPEND src_lib_shared_nupiccore_link_libraries ${src_common_os_libs}) +list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) -add_library(${src_lib_shared_nupiccore} SHARED - ${src_nupiccore_internal_srcs} - ${src_core_py_support_srcs}) +add_library(${_SRC_LIB_SHARED_NUPICCORE} SHARED + ${_SRC_NUPICCORE_INTERNAL_SRCS} + ${_SRC_CORE_PY_SUPPORT_SRCS}) -target_link_libraries(${src_lib_shared_nupiccore} - PRIVATE ${src_lib_shared_nupiccore_link_libraries}) +target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} + PRIVATE ${_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES}) # NOTE We use INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols that # have default visibility. This is necessary for the Windows build, where # symbols default to hidden visibility. -set_target_properties(${src_lib_shared_nupiccore} PROPERTIES COMPILE_FLAGS +set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES COMPILE_FLAGS "${src_nupiccore_internals_compile_flags} ${INTERNAL_CXX_FLAGS_NO_HIDE}") if(${NUPIC_IWYU}) # TODO: Create a target that doesn't include the generated capnp schema files # since we don't want to run iwyu on them and iwyu can only be applied to # CMake targets (not source files directly). - set_target_properties(${src_lib_shared_nupiccore} + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() @@ -489,12 +489,12 @@ if (NUPIC_BUILD_PYEXT_MODULES) # without resolving all symbols. This addresses resolution of python symbols # that will be provided by the python runtime as well as capnproto symbols # that will be provided by pycapnp. - set_target_properties(${src_lib_shared_nupiccore} PROPERTIES LINK_FLAGS + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") else() # When building a standalone shared library, specifically don't allow link # with undefined symbols - set_target_properties(${src_lib_shared_nupiccore} PROPERTIES LINK_FLAGS + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS "${INTERNAL_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") endif() @@ -504,14 +504,14 @@ if (APPLE) # This is specific to OS X; linux simply requires setting of RPATH in the # client binary. NOTE The RPATH mechanism facilitates relocatable installation # of packages. - set_target_properties(${src_lib_shared_nupiccore} PROPERTIES MACOSX_RPATH ON) + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES MACOSX_RPATH ON) endif() if (NUPIC_BUILD_PYEXT_MODULES AND PY_EXTENSIONS_DIR) # Request installtion into nupic.bindings's tree. We install it into the same # directory as our extensions in order to facilitate relocatable loading on # Windows, which doesn't support the RPATH mechanism. - install(TARGETS ${src_lib_shared_nupiccore} + install(TARGETS ${_SRC_LIB_SHARED_NUPICCORE} DESTINATION ${PY_EXTENSIONS_DIR}) endif() @@ -533,7 +533,7 @@ add_library(${src_lib_static_py_binding_support} STATIC # We rely on artifacts from building of nupic_core shared lib (e.g., # capnp schema C++ headers) add_dependencies(${src_lib_static_py_binding_support} - ${src_lib_shared_nupiccore}) + ${_SRC_LIB_SHARED_NUPICCORE}) set_target_properties(${src_lib_static_py_binding_support} PROPERTIES COMPILE_FLAGS ${src_nupiccore_internals_compile_flags}) @@ -554,7 +554,7 @@ endif() # Common libs for test executables set(src_common_test_exe_libs ${src_lib_static_py_binding_support} - ${src_lib_shared_nupiccore} + ${_SRC_LIB_SHARED_NUPICCORE} ${PYTHON_LIBRARIES} ${src_common_os_libs}) @@ -858,7 +858,7 @@ if (NUPIC_BUILD_PYEXT_MODULES) # in our own compilation of capnproto on objects created by pycapnp's. set(_SRC_SWIG_LINK_LIBRARIES ${src_lib_static_py_binding_support} - ${src_lib_shared_nupiccore} + ${_SRC_LIB_SHARED_NUPICCORE} ${src_common_os_libs}) # Common dependencies for our python extensions for use with @@ -1017,7 +1017,7 @@ endif() # NUPIC_BUILD_PYEXT_MODULES # Install targets into CMAKE_INSTALL_PREFIX # install(TARGETS - ${src_lib_shared_nupiccore} + ${_SRC_LIB_SHARED_NUPICCORE} ${src_lib_static_gtest} ${src_executable_helloregion} ${src_executable_cppregiontest} @@ -1032,7 +1032,7 @@ install(TARGETS # Install a copy of nupic_core shared lib into bin to facilitate running tests # in bin that depend on it. -install(TARGETS ${src_lib_shared_nupiccore} +install(TARGETS ${_SRC_LIB_SHARED_NUPICCORE} DESTINATION bin) From 83321baf2051cab2d5cfd939139bc1275740e17d Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 27 Jan 2017 20:30:31 -0800 Subject: [PATCH 17/44] NUP-2320 Only wrap specific archives in "WHOLE ARCHIVE". --- src/CMakeLists.txt | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f3637b54f..d56345458d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,13 +58,15 @@ endif() # # Linker options # -if(MSVC) - set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "/WHOLEARCHIVE") -elseif(APPLE) - set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "-all_load") -else() +if(APPLE) + set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") + set(_SRC_WHOLE_ARCHIVE_END) +elseif(CMAKE_COMPILER_IS_GNUCXX) # Linux and MINGW - set(_SRC_WHOLE_ARCHIVE_LINKER_OPTION "-Wl,--whole-archive") + set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") + set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") +else() + message (FATAL_ERROR "Unsupported compiler!") endif() @@ -459,7 +461,9 @@ elseif("${PLATFORM}" STREQUAL "windows") # nupic/nupic.bindings presently have no dependency on pycapnp. list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES} - ${CAPNP_STATIC_LIB_TARGET}) + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END}) endif() list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) @@ -468,8 +472,8 @@ add_library(${_SRC_LIB_SHARED_NUPICCORE} SHARED ${_SRC_NUPICCORE_INTERNAL_SRCS} ${_SRC_CORE_PY_SUPPORT_SRCS}) -target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} - PRIVATE ${_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES}) +target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} PRIVATE + ${_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES}) # NOTE We use INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols that # have default visibility. This is necessary for the Windows build, where @@ -490,12 +494,12 @@ if (NUPIC_BUILD_PYEXT_MODULES) # that will be provided by the python runtime as well as capnproto symbols # that will be provided by pycapnp. set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS - "${PYEXT_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") + "${PYEXT_LINKER_FLAGS_OPTIMIZED}") else() # When building a standalone shared library, specifically don't allow link # with undefined symbols set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS - "${INTERNAL_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") + ${INTERNAL_LINKER_FLAGS_OPTIMIZED}) endif() if (APPLE) @@ -568,11 +572,14 @@ if (NUPIC_BUILD_PYEXT_MODULES) set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES COMPILE_FLAGS "${src_nupiccore_internals_compile_flags}") - target_link_libraries(${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE ${CAPNP_STATIC_LIB_TARGET}) + target_link_libraries(${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END}) # Make sure the entire capnproto archive is added to the shared lib set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES LINK_FLAGS - "${INTERNAL_LINKER_FLAGS_OPTIMIZED} ${_SRC_WHOLE_ARCHIVE_LINKER_OPTION}") + ${INTERNAL_LINKER_FLAGS_OPTIMIZED}) if (APPLE) set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES MACOSX_RPATH ON) From 074671200e1e2955696b9a03e7ed874311f30b99 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 10:51:27 -0800 Subject: [PATCH 18/44] NUP-2320 Link lib capnproto_shared with src_common_os_libs to satisify dependency on pthread. --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d56345458d..1dee9807a4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -575,7 +575,8 @@ if (NUPIC_BUILD_PYEXT_MODULES) target_link_libraries(${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE ${_SRC_WHOLE_ARCHIVE_START} ${CAPNP_STATIC_LIB_TARGET} - ${_SRC_WHOLE_ARCHIVE_END}) + ${_SRC_WHOLE_ARCHIVE_END} + src_common_os_libs) # Make sure the entire capnproto archive is added to the shared lib set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES LINK_FLAGS From 8ef6762da1e3f266b336dc87750fec4ba1c60457 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 11:15:46 -0800 Subject: [PATCH 19/44] NUP-2320 Define _SRC_WHOLE_ARCHIVE_START for Clang. NUP-2320 Fix reference to src_common_os_libs variable. --- src/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1dee9807a4..9898cef58b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,15 +58,15 @@ endif() # # Linker options # -if(APPLE) +if(${APPLE) set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") set(_SRC_WHOLE_ARCHIVE_END) -elseif(CMAKE_COMPILER_IS_GNUCXX) +elseif(CMAKE_COMPILER_IS_GNUCXX OR "${PLATFORM}" STREQUAL "linux") # Linux and MINGW set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") else() - message (FATAL_ERROR "Unsupported compiler!") + message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") endif() @@ -576,7 +576,7 @@ if (NUPIC_BUILD_PYEXT_MODULES) ${_SRC_WHOLE_ARCHIVE_START} ${CAPNP_STATIC_LIB_TARGET} ${_SRC_WHOLE_ARCHIVE_END} - src_common_os_libs) + ${src_common_os_libs}) # Make sure the entire capnproto archive is added to the shared lib set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES LINK_FLAGS From 183acd0d4e0e15286f9f251208f9320af04958bf Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 11:20:52 -0800 Subject: [PATCH 20/44] fixed APPLE reference. --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9898cef58b..859af7463a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,11 +58,11 @@ endif() # # Linker options # -if(${APPLE) +if(${APPLE}) set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") set(_SRC_WHOLE_ARCHIVE_END) elseif(CMAKE_COMPILER_IS_GNUCXX OR "${PLATFORM}" STREQUAL "linux") - # Linux and MINGW + # MINGW or any compiler on linux set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") else() From 504de1f3a23535faa4757cb1d9f96c88ad7cea78 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 14:15:22 -0800 Subject: [PATCH 21/44] NUP-2320 Try a different WHOLEARCHIVE option for windows --- src/CMakeLists.txt | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 859af7463a..e6301c9638 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,13 +58,16 @@ endif() # # Linker options # -if(${APPLE}) +if(APPLE) set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") set(_SRC_WHOLE_ARCHIVE_END) -elseif(CMAKE_COMPILER_IS_GNUCXX OR "${PLATFORM}" STREQUAL "linux") - # MINGW or any compiler on linux - set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") - set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") +elseif(WIN32) + set(_SRC_WHOLE_ARCHIVE_START "/WHOLEARCHIVE:") + set(_SRC_WHOLE_ARCHIVE_END) +elseif("${PLATFORM}" STREQUAL "linux") + # Any compiler on linux + set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive ") + set(_SRC_WHOLE_ARCHIVE_END " -Wl,--no-whole-archive") else() message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") endif() @@ -461,13 +464,14 @@ elseif("${PLATFORM}" STREQUAL "windows") # nupic/nupic.bindings presently have no dependency on pycapnp. list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES} - ${_SRC_WHOLE_ARCHIVE_START} - ${CAPNP_STATIC_LIB_TARGET} - ${_SRC_WHOLE_ARCHIVE_END}) + ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END}) endif() list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) +message(STATUS "_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES = ${_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES}") + + add_library(${_SRC_LIB_SHARED_NUPICCORE} SHARED ${_SRC_NUPICCORE_INTERNAL_SRCS} ${_SRC_CORE_PY_SUPPORT_SRCS}) @@ -572,11 +576,15 @@ if (NUPIC_BUILD_PYEXT_MODULES) set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES COMPILE_FLAGS "${src_nupiccore_internals_compile_flags}") - target_link_libraries(${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE - ${_SRC_WHOLE_ARCHIVE_START} - ${CAPNP_STATIC_LIB_TARGET} - ${_SRC_WHOLE_ARCHIVE_END} - ${src_common_os_libs}) + set(_SRC_SHARED_CAPNPROTO_LIBS + ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END} + ${src_common_os_libs}) + + message(STATUS "_SRC_SHARED_CAPNPROTO_LIBS = ${_SRC_SHARED_CAPNPROTO_LIBS}") + + target_link_libraries( + ${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE + ${_SRC_SHARED_CAPNPROTO_LIBS}) # Make sure the entire capnproto archive is added to the shared lib set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES LINK_FLAGS From 9f83b8334dff4cca6ce856772ee5b64bf40d6b66 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 15:42:48 -0800 Subject: [PATCH 22/44] Revert syntax of "whole archive" flags. The builds were unhappy with the prior attempt. --- src/CMakeLists.txt | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e6301c9638..dd5004072c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -59,15 +59,17 @@ endif() # Linker options # if(APPLE) + # Any compiler on OS X set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") set(_SRC_WHOLE_ARCHIVE_END) elseif(WIN32) + # Any compiler on Windows set(_SRC_WHOLE_ARCHIVE_START "/WHOLEARCHIVE:") set(_SRC_WHOLE_ARCHIVE_END) elseif("${PLATFORM}" STREQUAL "linux") - # Any compiler on linux - set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive ") - set(_SRC_WHOLE_ARCHIVE_END " -Wl,--no-whole-archive") + # Any compiler on Linux + set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") + set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") else() message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") endif() @@ -459,12 +461,14 @@ elseif("${PLATFORM}" STREQUAL "windows") # all symbols to resolve at link time, so we have to add libpython for this # platform # - # NOTE On Windows builds, we include capnproto because we - # presently build self-contained CAPNP_LITE on Windows, and Windows - # nupic/nupic.bindings presently have no dependency on pycapnp. + # NOTE On Windows builds, we include capnproto because we presently build + # self-contained CAPNP_LITE on Windows, and Windows nupic/nupic.bindings + # presently have no dependency on pycapnp. list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES} - ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END}) + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END}) endif() list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) @@ -567,17 +571,19 @@ set(src_common_test_exe_libs ${src_common_os_libs}) # Add capnproto lib to our C/C++ test apps when building python -# extensions, since capnproto objects are excluded from nupic_core shared lib -# in this case to avoid conflict with capnproto methods compiled into the -# pycapnp extension -if (NUPIC_BUILD_PYEXT_MODULES) +# extensions, since capnproto objects are excluded from non-Nindows nupic_core +# shared lib in this case to avoid conflict with capnproto methods compiled into +# the pycapnp extension +if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) set(_SRC_LIB_SHARED_CAPNPROTO capnproto_shared) add_library(${_SRC_LIB_SHARED_CAPNPROTO} SHARED "empty.cpp") set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES COMPILE_FLAGS "${src_nupiccore_internals_compile_flags}") set(_SRC_SHARED_CAPNPROTO_LIBS - ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END} + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END} ${src_common_os_libs}) message(STATUS "_SRC_SHARED_CAPNPROTO_LIBS = ${_SRC_SHARED_CAPNPROTO_LIBS}") From 87b8bf75caf277d1a53fa48fec44c8485a0012ef Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 16:11:20 -0800 Subject: [PATCH 23/44] NUP-2320 Use same "whole archive" option for mingw as for Linux. --- src/CMakeLists.txt | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dd5004072c..3b9bfc7e5d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,18 +58,16 @@ endif() # # Linker options # -if(APPLE) - # Any compiler on OS X - set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") - set(_SRC_WHOLE_ARCHIVE_END) -elseif(WIN32) - # Any compiler on Windows +if(MSVC) set(_SRC_WHOLE_ARCHIVE_START "/WHOLEARCHIVE:") set(_SRC_WHOLE_ARCHIVE_END) -elseif("${PLATFORM}" STREQUAL "linux") - # Any compiler on Linux - set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") - set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") +elseif(APPLE) + set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") + set(_SRC_WHOLE_ARCHIVE_END) +elseif((CMAKE_COMPILER_IS_GNUCXX AND WIN32) OR "${PLATFORM}" STREQUAL "linux") + # GNU on Windows (e.g. MINGW) or any compiler on linux + set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive ") + set(_SRC_WHOLE_ARCHIVE_END " -Wl,--no-whole-archive") else() message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") endif() @@ -461,14 +459,12 @@ elseif("${PLATFORM}" STREQUAL "windows") # all symbols to resolve at link time, so we have to add libpython for this # platform # - # NOTE On Windows builds, we include capnproto because we presently build - # self-contained CAPNP_LITE on Windows, and Windows nupic/nupic.bindings - # presently have no dependency on pycapnp. + # NOTE On Windows builds, we include capnproto because we + # presently build self-contained CAPNP_LITE on Windows, and Windows + # nupic/nupic.bindings presently have no dependency on pycapnp. list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES} - ${_SRC_WHOLE_ARCHIVE_START} - ${CAPNP_STATIC_LIB_TARGET} - ${_SRC_WHOLE_ARCHIVE_END}) + ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END}) endif() list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) @@ -571,19 +567,17 @@ set(src_common_test_exe_libs ${src_common_os_libs}) # Add capnproto lib to our C/C++ test apps when building python -# extensions, since capnproto objects are excluded from non-Nindows nupic_core -# shared lib in this case to avoid conflict with capnproto methods compiled into -# the pycapnp extension -if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) +# extensions, since capnproto objects are excluded from nupic_core shared lib +# in this case to avoid conflict with capnproto methods compiled into the +# pycapnp extension +if (NUPIC_BUILD_PYEXT_MODULES) set(_SRC_LIB_SHARED_CAPNPROTO capnproto_shared) add_library(${_SRC_LIB_SHARED_CAPNPROTO} SHARED "empty.cpp") set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES COMPILE_FLAGS "${src_nupiccore_internals_compile_flags}") set(_SRC_SHARED_CAPNPROTO_LIBS - ${_SRC_WHOLE_ARCHIVE_START} - ${CAPNP_STATIC_LIB_TARGET} - ${_SRC_WHOLE_ARCHIVE_END} + ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END} ${src_common_os_libs}) message(STATUS "_SRC_SHARED_CAPNPROTO_LIBS = ${_SRC_SHARED_CAPNPROTO_LIBS}") From 02456372dca682ad024e3dbf2b07d9399520caa1 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 16:47:18 -0800 Subject: [PATCH 24/44] NUP-2320 Add `--export-all-symbols` link flag to GNUCXX-based builds. --- src/CMakeLists.txt | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b9bfc7e5d..1683787c5a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -454,7 +454,7 @@ if (NOT NUPIC_BUILD_PYEXT_MODULES) # environment, add dependency on python lib to resolve python symbol # references (e.g., from PyRegion) list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES}) -elseif("${PLATFORM}" STREQUAL "windows") +elseif(WIN32) # NOTE Windows DLLs are shared executables with their own main; they require # all symbols to resolve at link time, so we have to add libpython for this # platform @@ -464,7 +464,9 @@ elseif("${PLATFORM}" STREQUAL "windows") # nupic/nupic.bindings presently have no dependency on pycapnp. list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${PYTHON_LIBRARIES} - ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END}) + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END}) endif() list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) @@ -492,20 +494,30 @@ if(${NUPIC_IWYU}) PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() +set(_SRC_SHARED_NUPIC_CORE_LINK_FLAGS) + if (NUPIC_BUILD_PYEXT_MODULES) # When building a shared lib for use by our pythone extensions, allow building # without resolving all symbols. This addresses resolution of python symbols # that will be provided by the python runtime as well as capnproto symbols # that will be provided by pycapnp. - set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS - "${PYEXT_LINKER_FLAGS_OPTIMIZED}") + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${PYEXT_LINKER_FLAGS_OPTIMIZED}") else() # When building a standalone shared library, specifically don't allow link # with undefined symbols - set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS - ${INTERNAL_LINKER_FLAGS_OPTIMIZED}) + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${INTERNAL_LINKER_FLAGS_OPTIMIZED}") endif() +if(CMAKE_COMPILER_IS_GNUCXX) + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} --export-all-symbols") +endif() + +set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS + ${_SRC_SHARED_NUPICCORE_LINK_FLAGS}) + if (APPLE) # Enable MACOSX_RPATH to use @rpath as install_name of this shared lib to # allow client binary (app or lib) to locate it via the RPATH mechanism. @@ -566,18 +578,20 @@ set(src_common_test_exe_libs ${PYTHON_LIBRARIES} ${src_common_os_libs}) -# Add capnproto lib to our C/C++ test apps when building python +# Add capnproto lib to our C/C++ test apps when building non-Windows python # extensions, since capnproto objects are excluded from nupic_core shared lib # in this case to avoid conflict with capnproto methods compiled into the # pycapnp extension -if (NUPIC_BUILD_PYEXT_MODULES) +if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) set(_SRC_LIB_SHARED_CAPNPROTO capnproto_shared) add_library(${_SRC_LIB_SHARED_CAPNPROTO} SHARED "empty.cpp") set_target_properties(${_SRC_LIB_SHARED_CAPNPROTO} PROPERTIES COMPILE_FLAGS "${src_nupiccore_internals_compile_flags}") set(_SRC_SHARED_CAPNPROTO_LIBS - ${_SRC_WHOLE_ARCHIVE_START}${CAPNP_STATIC_LIB_TARGET}${_SRC_WHOLE_ARCHIVE_END} + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END} ${src_common_os_libs}) message(STATUS "_SRC_SHARED_CAPNPROTO_LIBS = ${_SRC_SHARED_CAPNPROTO_LIBS}") From 6e75c3872eaf91979ce685faef197e602f823810 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 16:48:19 -0800 Subject: [PATCH 25/44] NUP-2320 Log value of _SRC_SHARED_NUPICCORE_LINK_FLAGS --- src/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1683787c5a..b3e0fc1f03 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -515,6 +515,8 @@ if(CMAKE_COMPILER_IS_GNUCXX) "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} --export-all-symbols") endif() +message(STATUS "_SRC_SHARED_NUPICCORE_LINK_FLAGS=${_SRC_SHARED_NUPICCORE_LINK_FLAGS}") + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS ${_SRC_SHARED_NUPICCORE_LINK_FLAGS}) From 7ea90f634c6963d1f898c33b1a5591a4530e3690 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 16:52:54 -0800 Subject: [PATCH 26/44] NUP-2320 remove leading/trailing spaces from whole-archive linker flags. --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b3e0fc1f03..6d264fe101 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -66,8 +66,8 @@ elseif(APPLE) set(_SRC_WHOLE_ARCHIVE_END) elseif((CMAKE_COMPILER_IS_GNUCXX AND WIN32) OR "${PLATFORM}" STREQUAL "linux") # GNU on Windows (e.g. MINGW) or any compiler on linux - set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive ") - set(_SRC_WHOLE_ARCHIVE_END " -Wl,--no-whole-archive") + set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") + set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") else() message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") endif() From 2c98a61e05364fc4d9a3a8c851ddbe8211f49af8 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 30 Jan 2017 17:08:18 -0800 Subject: [PATCH 27/44] NUP-2320 No need for --export-all-symbols, since GCC automatically exports all global symbols. --- src/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d264fe101..e24c0ac0c3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -510,11 +510,6 @@ else() "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${INTERNAL_LINKER_FLAGS_OPTIMIZED}") endif() -if(CMAKE_COMPILER_IS_GNUCXX) - set(_SRC_SHARED_NUPICCORE_LINK_FLAGS - "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} --export-all-symbols") -endif() - message(STATUS "_SRC_SHARED_NUPICCORE_LINK_FLAGS=${_SRC_SHARED_NUPICCORE_LINK_FLAGS}") set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES LINK_FLAGS From 47ecbd6494f0b81b6de6919dce69d82bdccae4b9 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 31 Jan 2017 11:11:29 -0800 Subject: [PATCH 28/44] NUP-2320 Move _SRC_LIB_SHARED_CAPNPROTO after _SRC_LIB_SHARED_NUPICCORE in src_common_test_exe_libs. NUP-2320 experimentally, comment out the PRIVATE keyword in target_link_libraries of shared libs. --- src/CMakeLists.txt | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e24c0ac0c3..e16ab1d3c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -478,7 +478,7 @@ add_library(${_SRC_LIB_SHARED_NUPICCORE} SHARED ${_SRC_NUPICCORE_INTERNAL_SRCS} ${_SRC_CORE_PY_SUPPORT_SRCS}) -target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} PRIVATE +target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} # PRIVATE ${_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES}) # NOTE We use INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols that @@ -569,16 +569,12 @@ endif() # # Common libs for test executables -set(src_common_test_exe_libs - ${src_lib_static_py_binding_support} - ${_SRC_LIB_SHARED_NUPICCORE} - ${PYTHON_LIBRARIES} - ${src_common_os_libs}) -# Add capnproto lib to our C/C++ test apps when building non-Windows python -# extensions, since capnproto objects are excluded from nupic_core shared lib -# in this case to avoid conflict with capnproto methods compiled into the +# Create capnproto shared lib for our C/C++ test apps when building non-Windows +# python extensions, since capnproto objects are excluded from nupic_core shared +# lib in this case to avoid conflict with capnproto methods compiled into the # pycapnp extension +set(_SRC_LIB_SHARED_CAPNPROTO) if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) set(_SRC_LIB_SHARED_CAPNPROTO capnproto_shared) add_library(${_SRC_LIB_SHARED_CAPNPROTO} SHARED "empty.cpp") @@ -594,7 +590,7 @@ if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) message(STATUS "_SRC_SHARED_CAPNPROTO_LIBS = ${_SRC_SHARED_CAPNPROTO_LIBS}") target_link_libraries( - ${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE + ${_SRC_LIB_SHARED_CAPNPROTO} # PRIVATE ${_SRC_SHARED_CAPNPROTO_LIBS}) # Make sure the entire capnproto archive is added to the shared lib @@ -607,12 +603,17 @@ if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) install(TARGETS ${_SRC_LIB_SHARED_CAPNPROTO} DESTINATION bin) - - set(src_common_test_exe_libs - ${_SRC_LIB_SHARED_CAPNPROTO} - ${src_common_test_exe_libs}) endif() +set(src_common_test_exe_libs + ${_SRC_WHOLE_ARCHIVE_START} + ${src_lib_static_py_binding_support} + ${_SRC_WHOLE_ARCHIVE_END} + ${_SRC_LIB_SHARED_NUPICCORE} + ${_SRC_LIB_SHARED_CAPNPROTO} + ${PYTHON_LIBRARIES} + ${src_common_os_libs}) + message(STATUS "src_common_test_exe_libs = ${src_common_test_exe_libs}") From 598e1485edafadea66567f49c60753520aa7b534 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 31 Jan 2017 22:44:45 -0800 Subject: [PATCH 29/44] NUP-2320 Enable export of all symbols when building nupic_core DLL on windows. --- src/CMakeLists.txt | 52 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e16ab1d3c9..3ad220ef99 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,7 +64,7 @@ if(MSVC) elseif(APPLE) set(_SRC_WHOLE_ARCHIVE_START "-Wl,-force_load") set(_SRC_WHOLE_ARCHIVE_END) -elseif((CMAKE_COMPILER_IS_GNUCXX AND WIN32) OR "${PLATFORM}" STREQUAL "linux") +elseif(MINGW OR "${PLATFORM}" STREQUAL "linux") # GNU on Windows (e.g. MINGW) or any compiler on linux set(_SRC_WHOLE_ARCHIVE_START "-Wl,--whole-archive") set(_SRC_WHOLE_ARCHIVE_END "-Wl,--no-whole-archive") @@ -442,14 +442,12 @@ set(_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${APRUTIL1_STATIC_LIB_TARGET} ${Z_STATIC_LIB_TARGET}) -if (NOT NUPIC_BUILD_PYEXT_MODULES) +if(NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run without our python # extensions, add capnproto static lib. Our python extensions have their own # logic governing whether to include capnproto. list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${CAPNP_STATIC_LIB_TARGET}) -endif() -if (NOT NUPIC_BUILD_PYEXT_MODULES) # When building a standalone shared lib that will run outside the python # environment, add dependency on python lib to resolve python symbol # references (e.g., from PyRegion) @@ -462,11 +460,16 @@ elseif(WIN32) # NOTE On Windows builds, we include capnproto because we # presently build self-contained CAPNP_LITE on Windows, and Windows # nupic/nupic.bindings presently have no dependency on pycapnp. - list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES - ${PYTHON_LIBRARIES} - ${_SRC_WHOLE_ARCHIVE_START} - ${CAPNP_STATIC_LIB_TARGET} - ${_SRC_WHOLE_ARCHIVE_END}) + + if(MINGW) + list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES + ${_SRC_WHOLE_ARCHIVE_START} + ${CAPNP_STATIC_LIB_TARGET} + ${_SRC_WHOLE_ARCHIVE_END} + ${PYTHON_LIBRARIES}) + else() + message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") + endif() endif() list(APPEND _SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES ${src_common_os_libs}) @@ -494,15 +497,32 @@ if(${NUPIC_IWYU}) PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() -set(_SRC_SHARED_NUPIC_CORE_LINK_FLAGS) +set(_SRC_SHARED_NUPICCORE_LINK_FLAGS) + +if(WIN32) + if(MINGW) + # Turn on explicit -export-all-symbols on MINGW. The implicit one is + # disabled if any symbol in any object is marked with the + # __declspec(dllexport) attribute + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} -Wl,-export-all-symbols") + endif() +endif() if (NUPIC_BUILD_PYEXT_MODULES) - # When building a shared lib for use by our pythone extensions, allow building - # without resolving all symbols. This addresses resolution of python symbols - # that will be provided by the python runtime as well as capnproto symbols - # that will be provided by pycapnp. - set(_SRC_SHARED_NUPICCORE_LINK_FLAGS - "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${PYEXT_LINKER_FLAGS_OPTIMIZED}") + if(WIN32) + # On Windows, a self-contained DLL executable is built, so it has to resolve + # all symbols + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${INTERNAL_LINKER_FLAGS_OPTIMIZED}") + else() + # When building a shared lib for use by our pythone extensions, allow building + # without resolving all symbols. This addresses resolution of python symbols + # that will be provided by the python runtime as well as capnproto symbols + # that will be provided by pycapnp. + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${PYEXT_LINKER_FLAGS_OPTIMIZED}") + endif() else() # When building a standalone shared library, specifically don't allow link # with undefined symbols From 60207ae3c33b734ff16d685eff632e22701fe2e0 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 31 Jan 2017 23:03:58 -0800 Subject: [PATCH 30/44] NUP-2320 Use WINDOWS_EXPORT_ALL_SYMBOLS instead of MINGW-specific -export-all-symbols. --- src/CMakeLists.txt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3ad220ef99..efacf5fb14 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -497,18 +497,16 @@ if(${NUPIC_IWYU}) PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() -set(_SRC_SHARED_NUPICCORE_LINK_FLAGS) - if(WIN32) - if(MINGW) - # Turn on explicit -export-all-symbols on MINGW. The implicit one is - # disabled if any symbol in any object is marked with the - # __declspec(dllexport) attribute - set(_SRC_SHARED_NUPICCORE_LINK_FLAGS - "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} -Wl,-export-all-symbols") - endif() + # Turn on explicit export of all symbols. The implicit one is + # disabled by MINGW if any symbol in any object is marked with the + # __declspec(dllexport) attribute + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS ON) endif() +set(_SRC_SHARED_NUPICCORE_LINK_FLAGS) + if (NUPIC_BUILD_PYEXT_MODULES) if(WIN32) # On Windows, a self-contained DLL executable is built, so it has to resolve From 24955c0bad42811aa349c28c58f90d6ce1ce2e17 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 31 Jan 2017 23:18:27 -0800 Subject: [PATCH 31/44] Revert "NUP-2320 Use WINDOWS_EXPORT_ALL_SYMBOLS instead of MINGW-specific -export-all-symbols." This reverts commit 045e0bf7423c6dc56153862c5b8bfdb3626d3352. --- src/CMakeLists.txt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index efacf5fb14..3ad220ef99 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -497,16 +497,18 @@ if(${NUPIC_IWYU}) PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) endif() +set(_SRC_SHARED_NUPICCORE_LINK_FLAGS) + if(WIN32) - # Turn on explicit export of all symbols. The implicit one is - # disabled by MINGW if any symbol in any object is marked with the - # __declspec(dllexport) attribute - set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES - WINDOWS_EXPORT_ALL_SYMBOLS ON) + if(MINGW) + # Turn on explicit -export-all-symbols on MINGW. The implicit one is + # disabled if any symbol in any object is marked with the + # __declspec(dllexport) attribute + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} -Wl,-export-all-symbols") + endif() endif() -set(_SRC_SHARED_NUPICCORE_LINK_FLAGS) - if (NUPIC_BUILD_PYEXT_MODULES) if(WIN32) # On Windows, a self-contained DLL executable is built, so it has to resolve From 0e5d18e4881674a7ee515448eb4f5038e0eccad3 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Tue, 31 Jan 2017 23:20:43 -0800 Subject: [PATCH 32/44] NUP-2320 Support "export all symbols" for MSVC as well as MINGW, using compiler-specific techniques. --- src/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3ad220ef99..e21c09164d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -506,6 +506,11 @@ if(WIN32) # __declspec(dllexport) attribute set(_SRC_SHARED_NUPICCORE_LINK_FLAGS "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} -Wl,-export-all-symbols") + elseif(MSVC) + set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS ON) + else() + message (FATAL_ERROR "Unsupported compiler! CMAKE_CXX_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}") endif() endif() From c5c33d79ac5f42e75eb9b995b2ca647b16912bcb Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 1 Feb 2017 13:39:40 -0800 Subject: [PATCH 33/44] NUP-2320 On GNU builds - including MINGW - always link with -shared-libgcc (not static) to be able to handle exceptions across DLL boundaries. --- CommonCompilerConfig.cmake | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index eee48d33f4..ce16b67ff4 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -191,6 +191,11 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") endif() if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + # NOTE We need to use shared libgcc to be able to throw and catch exceptions + # across different shared libraries, as may be the case when our python + # extensions runtime-link to capnproto symbols in pycapnp's extension. + set(stdlib_common "${stdlib_common} -shared-libgcc") + if (${NUPIC_BUILD_PYEXT_MODULES} AND "${PLATFORM}" STREQUAL "linux") # NOTE When building manylinux python extensions, we want the static # libstdc++ due to differences in c++ ABI between the older toolchain in the @@ -199,13 +204,7 @@ if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") # manylinux-built extension is unable to catch std::ios::failure exception # raised by the shared libstdc++.so while running on Ubuntu 16.04. set(stdlib_cxx "${stdlib_cxx} -static-libstdc++") - - # NOTE We need to use shared libgcc to be able to throw and catch exceptions - # across different shared libraries, as may be the case when our python - # extensions runtime-link to capnproto symbols in pycapnp's extension. - set(stdlib_common "${stdlib_common} -shared-libgcc") else() - set(stdlib_common "${stdlib_common} -static-libgcc") set(stdlib_cxx "${stdlib_cxx} -static-libstdc++") endif() endif() From d7929fa630079075dfde3320650308c4c78e0e46 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 1 Feb 2017 14:36:05 -0800 Subject: [PATCH 34/44] NUP-2320 Added missing advapi32 to the link list for MINGW builds. Its absence was causing a load failure of py_region_test.exe on a Win32 AMI on Amazon. --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e21c09164d..c3ebc1995e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -309,7 +309,7 @@ if("${PLATFORM}" STREQUAL "linux") elseif("${PLATFORM}" STREQUAL "darwin") list(APPEND src_common_os_libs c++abi) elseif(MSYS OR MINGW) - list(APPEND src_common_os_libs psapi ws2_32 wsock32 rpcrt4) + list(APPEND src_common_os_libs advapi32 psapi ws2_32 wsock32 rpcrt4) elseif("${PLATFORM}" STREQUAL "windows") list(APPEND src_common_os_libs oldnames.lib psapi.lib ws2_32.lib) endif() From df1790d0022e3504fd450c819ed5c9301472091b Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 1 Feb 2017 16:00:46 -0800 Subject: [PATCH 35/44] NUP-2320 Forego installation of a plethora of header files when building nupic.bindings. --- src/CMakeLists.txt | 63 +++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c3ebc1995e..1a0779bbde 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1088,47 +1088,48 @@ install(TARGETS install(TARGETS ${_SRC_LIB_SHARED_NUPICCORE} DESTINATION bin) +if (NOT NUPIC_BUILD_PYEXT_MODULES) + install(DIRECTORY nupic/ DESTINATION include/nupic + FILES_MATCHING PATTERN "*.h*" + PATTERN "*.hpp.in" EXCLUDE) + install(DIRECTORY nupic/ DESTINATION include/nupic + FILES_MATCHING PATTERN "*.capnp") + install(DIRECTORY ${PROJECT_BINARY_DIR}/nupic/ DESTINATION include/nupic/ + FILES_MATCHING PATTERN "*.capnp.h") + + install(DIRECTORY ${REPOSITORY_DIR}/external/common/include/gtest + DESTINATION include/gtest + FILES_MATCHING PATTERN "*.h*") + + install(FILES ${PROJECT_BINARY_DIR}/Version.hpp + DESTINATION include/nupic/) + + install(DIRECTORY "${REPOSITORY_DIR}/external/common/include/" + DESTINATION include) -install(DIRECTORY nupic/ DESTINATION include/nupic - FILES_MATCHING PATTERN "*.h*" - PATTERN "*.hpp.in" EXCLUDE) -install(DIRECTORY nupic/ DESTINATION include/nupic - FILES_MATCHING PATTERN "*.capnp") -install(DIRECTORY ${PROJECT_BINARY_DIR}/nupic/ DESTINATION include/nupic/ - FILES_MATCHING PATTERN "*.capnp.h") - -install(DIRECTORY ${REPOSITORY_DIR}/external/common/include/gtest - DESTINATION include/gtest - FILES_MATCHING PATTERN "*.h*") - -install(FILES ${PROJECT_BINARY_DIR}/Version.hpp - DESTINATION include/nupic/) - -install(DIRECTORY "${REPOSITORY_DIR}/external/common/include/" - DESTINATION include) + install(DIRECTORY "${src_numpy_core}/include/" + DESTINATION include) -install(DIRECTORY "${src_numpy_core}/lib/" - DESTINATION lib) -install(DIRECTORY "${src_numpy_core}/include/" - DESTINATION include) + foreach(directory ${CAPNP_INCLUDE_DIRS}) + install(DIRECTORY "${directory}/capnp" + DESTINATION include) + install(DIRECTORY "${directory}/kj" + DESTINATION include) + endforeach() -foreach(directory ${CAPNP_INCLUDE_DIRS}) - install(DIRECTORY "${directory}/capnp" - DESTINATION include) - install(DIRECTORY "${directory}/kj" - DESTINATION include) -endforeach() + install(DIRECTORY nupic/py_support DESTINATION include/nupic + FILES_MATCHING PATTERN "*.c*") + install(DIRECTORY nupic DESTINATION include/nupic + FILES_MATCHING PATTERN "*.py") -install(DIRECTORY nupic/py_support DESTINATION include/nupic - FILES_MATCHING PATTERN "*.c*") -install(DIRECTORY nupic DESTINATION include/nupic - FILES_MATCHING PATTERN "*.py") +endif(NOT NUPIC_BUILD_PYEXT_MODULES) # # `make package` results in # nupic_core-${NUPIC_CORE_VERSION}-${PLATFORM}${BITNESS}${PLATFORM_SUFFIX}.tar.gz binary release # + set(CPACK_GENERATOR "TGZ") set(CPACK_PACKAGE_FILE_NAME "nupic_core-${NUPIC_CORE_VERSION}-${PLATFORM}${BITNESS}${PLATFORM_SUFFIX}") include(CPack) From 3783395785b60a3a6c76dcf8e7a14711199916af Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 1 Feb 2017 16:23:46 -0800 Subject: [PATCH 36/44] NUP-2320 Mark nupic-core's link librar interface as PRIVATE. --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a0779bbde..29a68ed0fd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -481,7 +481,7 @@ add_library(${_SRC_LIB_SHARED_NUPICCORE} SHARED ${_SRC_NUPICCORE_INTERNAL_SRCS} ${_SRC_CORE_PY_SUPPORT_SRCS}) -target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} # PRIVATE +target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} PRIVATE ${_SRC_LIB_SHARED_NUPICCORE_LINK_LIBRARIES}) # NOTE We use INTERNAL_CXX_FLAGS_NO_HIDE to force visibility of symbols that @@ -615,7 +615,7 @@ if (NUPIC_BUILD_PYEXT_MODULES AND NOT WIN32) message(STATUS "_SRC_SHARED_CAPNPROTO_LIBS = ${_SRC_SHARED_CAPNPROTO_LIBS}") target_link_libraries( - ${_SRC_LIB_SHARED_CAPNPROTO} # PRIVATE + ${_SRC_LIB_SHARED_CAPNPROTO} PRIVATE ${_SRC_SHARED_CAPNPROTO_LIBS}) # Make sure the entire capnproto archive is added to the shared lib From aae56443dfb84a07d6b2a10de4b0f705de334a66 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Wed, 1 Feb 2017 17:06:23 -0800 Subject: [PATCH 37/44] NUP-2320 Added printfs to help debug py_region_test failures on some targets. --- src/test/integration/PyRegionTest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/integration/PyRegionTest.cpp b/src/test/integration/PyRegionTest.cpp index 989eb7b673..432308454d 100644 --- a/src/test/integration/PyRegionTest.cpp +++ b/src/test/integration/PyRegionTest.cpp @@ -497,9 +497,11 @@ int realmain(bool leakTest) // should fail because network has not been initialized SHOULDFAIL(n.run(1)); + std::cerr << "ZZZ Survived SHOULDFAIL(n.run(1))" << std::endl << std::flush; // should fail because network can't be initialized SHOULDFAIL (n.initialize() ); + std::cerr << "ZZZ Survived SHOULDFAIL (n.initialize() )" << std::endl << std::flush; std::cout << "Setting dimensions of level1..." << std::endl; Dimensions d; @@ -512,6 +514,7 @@ int realmain(bool leakTest) std::cout << "Initializing again..." << std::endl; n.initialize(); + std::cerr << "ZZZ Survived n.initialize()" << std::endl << std::flush; testExceptionBug(); testPynodeInputOutputAccess(level2); From 486be62b098dc865d767ea148b9d3a113f989e21 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Thu, 2 Feb 2017 11:37:34 -0800 Subject: [PATCH 38/44] Added `BEFORE SHOULDFAIL(n.run(1))` to make sure we're able to get some output. --- src/test/integration/PyRegionTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/integration/PyRegionTest.cpp b/src/test/integration/PyRegionTest.cpp index 432308454d..f2a780f2f6 100644 --- a/src/test/integration/PyRegionTest.cpp +++ b/src/test/integration/PyRegionTest.cpp @@ -496,6 +496,7 @@ int realmain(bool leakTest) NTA_CHECK(rval == 77.7); // should fail because network has not been initialized + std::cerr << "ZZZ BEFORE SHOULDFAIL(n.run(1))" << std::endl << std::flush; SHOULDFAIL(n.run(1)); std::cerr << "ZZZ Survived SHOULDFAIL(n.run(1))" << std::endl << std::flush; From 471959119545fb45db226ba89be741c7f162bd88 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Thu, 2 Feb 2017 12:53:40 -0800 Subject: [PATCH 39/44] Try with defualt libstdc++ and libgcc. --- CommonCompilerConfig.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index ce16b67ff4..e21e55fe93 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -194,7 +194,7 @@ if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") # NOTE We need to use shared libgcc to be able to throw and catch exceptions # across different shared libraries, as may be the case when our python # extensions runtime-link to capnproto symbols in pycapnp's extension. - set(stdlib_common "${stdlib_common} -shared-libgcc") + #set(stdlib_common "${stdlib_common} -shared-libgcc") if (${NUPIC_BUILD_PYEXT_MODULES} AND "${PLATFORM}" STREQUAL "linux") # NOTE When building manylinux python extensions, we want the static @@ -203,9 +203,9 @@ if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") # compiled with the c++11 ABI. for example, with shared libstdc++, the # manylinux-built extension is unable to catch std::ios::failure exception # raised by the shared libstdc++.so while running on Ubuntu 16.04. - set(stdlib_cxx "${stdlib_cxx} -static-libstdc++") + #set(stdlib_cxx "${stdlib_cxx} -static-libstdc++") else() - set(stdlib_cxx "${stdlib_cxx} -static-libstdc++") + #set(stdlib_cxx "${stdlib_cxx} -static-libstdc++") endif() endif() From cf244f3df734aae9ceb191a4f7af8aa1d49a07dc Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Thu, 2 Feb 2017 16:32:12 -0800 Subject: [PATCH 40/44] NUP-2320 Limit symbols exported from nupic_core DLL when building extensions. --- src/CMakeLists.txt | 8 ++++++++ src/shared_nupic_core.map | 14 ++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 src/shared_nupic_core.map diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 29a68ed0fd..e7ec8e56d5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -515,7 +515,15 @@ if(WIN32) endif() if (NUPIC_BUILD_PYEXT_MODULES) + if ("${PLATFORM}" STREQUAL "linux" OR MINGW) + # Don't export more than needed + set(_SRC_SHARED_NUPICCORE_LINK_FLAGS + "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/shared_nupic_core.map") + add_dependencies(${_SRC_LIB_SHARED_NUPICCORE} ${CMAKE_CURRENT_SOURCE_DIR}/shared_nupic_core.map) + endif() + if(WIN32) + # On Windows, a self-contained DLL executable is built, so it has to resolve # all symbols set(_SRC_SHARED_NUPICCORE_LINK_FLAGS diff --git a/src/shared_nupic_core.map b/src/shared_nupic_core.map new file mode 100644 index 0000000000..907ba158d2 --- /dev/null +++ b/src/shared_nupic_core.map @@ -0,0 +1,14 @@ +{ + global: + apr_*; + extern "C++" { + *nupic::*; + nupic::*; + *capnp::*; + capnp::*; + *kj::*; + kj::* + }; + local: *; +}; + From a30831b821b4d78e498ddf16e2bc166ba394e662 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Thu, 2 Feb 2017 17:06:33 -0800 Subject: [PATCH 41/44] Add exported functions from ImageSensorLite.hpp to version script. --- src/shared_nupic_core.map | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/shared_nupic_core.map b/src/shared_nupic_core.map index 907ba158d2..0dc7569af8 100644 --- a/src/shared_nupic_core.map +++ b/src/shared_nupic_core.map @@ -1,6 +1,9 @@ { global: apr_*; + adjustBox; + accessPixels; + extractAuxInfo; extern "C++" { *nupic::*; nupic::*; From 67c6d424d2dcc8cd82b54053aaa0ce565c7bba65 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Thu, 2 Feb 2017 19:01:48 -0800 Subject: [PATCH 42/44] NUP-2320 Added gaborCompute to nupic_core's version script. --- src/shared_nupic_core.map | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shared_nupic_core.map b/src/shared_nupic_core.map index 0dc7569af8..d1479a5a0e 100644 --- a/src/shared_nupic_core.map +++ b/src/shared_nupic_core.map @@ -4,6 +4,7 @@ adjustBox; accessPixels; extractAuxInfo; + gaborCompute; extern "C++" { *nupic::*; nupic::*; From 00c9e51cd3f6509ca01549fc066f9c3c67bda451 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Fri, 3 Feb 2017 11:01:39 -0800 Subject: [PATCH 43/44] NUP-2320 Remove the experimental INTERNAL_CXX_FLAGS_NO_HIDE from nupic_core lib's LINK_FLAGS. --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e7ec8e56d5..893ce35156 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -488,7 +488,7 @@ target_link_libraries(${_SRC_LIB_SHARED_NUPICCORE} PRIVATE # have default visibility. This is necessary for the Windows build, where # symbols default to hidden visibility. set_target_properties(${_SRC_LIB_SHARED_NUPICCORE} PROPERTIES COMPILE_FLAGS - "${src_nupiccore_internals_compile_flags} ${INTERNAL_CXX_FLAGS_NO_HIDE}") + "${src_nupiccore_internals_compile_flags}") if(${NUPIC_IWYU}) # TODO: Create a target that doesn't include the generated capnp schema files # since we don't want to run iwyu on them and iwyu can only be applied to From 9ceb44083931ae5e12d30947dd1469cf11b1dd89 Mon Sep 17 00:00:00 2001 From: Vitaly Kruglikov Date: Mon, 6 Feb 2017 11:30:35 -0800 Subject: [PATCH 44/44] fix typo in comment --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 893ce35156..bd80c4ed50 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -529,7 +529,7 @@ if (NUPIC_BUILD_PYEXT_MODULES) set(_SRC_SHARED_NUPICCORE_LINK_FLAGS "${_SRC_SHARED_NUPICCORE_LINK_FLAGS} ${INTERNAL_LINKER_FLAGS_OPTIMIZED}") else() - # When building a shared lib for use by our pythone extensions, allow building + # When building a shared lib for use by our python extensions, allow building # without resolving all symbols. This addresses resolution of python symbols # that will be provided by the python runtime as well as capnproto symbols # that will be provided by pycapnp.