diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 737af891c9..72aa574dcd 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -91,6 +91,14 @@ jobs: name: "dist-${{ matrix.os }}" path: build/Release/distr/dist + - name: Test python package is working + shell: bash + run: | + python -m pip uninstall -y htm.core + filePath=$(ls build/Release/distr/dist/*.whl) + python -m pip install --force $filePath + python -m pytest bindings/py/tests + publish-pypi: name: Publish package to PYPI @@ -105,29 +113,32 @@ jobs: if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') with: name: dist-ubuntu-18.04 - path: dist1/ + path: dist/ - uses: actions/download-artifact@master if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') with: name: dist-macOs-latest - path: dist2/ + path: dist/ - uses: actions/download-artifact@master if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') with: name: dist-windows-2019 - path: dist3/ + path: dist/ - name: pre-PyPI if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') #copy dist data to /dist, where PyPI Action expects it run: | - ls dist* + cd dist + mv `ls -1 htm*-linux_*.whl` `ls -1 htm*-linux_*.whl | sed -e's/linux/manylinux1/'` #rename to manylinux1 to make PYPI accept the package + rm *.egg #remove obsoleted egg format + cd .. + ls dist/ - name: Publish to PyPI - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && matrix.os == 'ubuntu-18.04' && 1==0 - #FIXME temp disabled as pypi-publisher action must run on Linux only! + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@master with: user: __token__ diff --git a/.gitignore b/.gitignore index b16cfcda72..0935fe2319 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.swp *.pyc *.pkl +__pycache__ # Coverage files .coverage @@ -30,6 +31,7 @@ htm.core.kdev4 # Build files build/ junit-test-results.xml +VERSION # Test Output files bindings/py/tests/*.stream diff --git a/bindings/py/README.md b/bindings/py/README.md index e07fa9da54..c35ce4441d 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -1,33 +1,23 @@ ## Layout of directories: ``` - Repository - bindings + REPO_DIR external -- cmake scripts to download/build dependancies + bindings py -- Location of Python bindings packaging -- Contains things needed to build package cpp_src bindings -- C++ pybind11 mapping source code plugin -- C++ code to manage python plugin - tests -- Unit test for python interface + tests -- Unit test for python interface py htm -- python source code goes here tests src htm -- C++ source code goes here examples tests build -- where all things modified by the build are placed (disposible) ThirdParty -- where all thirdParty packages are installed. scripts -- temporary build files Release -- Where things are installed ``` ## This is where we build the distribution package: ``` - Repository + REPO_DIR build - scripts --- CMake build artifacts are in here. Release - bin --- Contains executables for unit tests and examples - include --- Include C++ header files - lib --- Static & Dynamic compiled libraries for htm core found here - distr --- Python distribution (packaging directory copied here) - setup.py - requirements.txt - dist --- pip egg is generated here - htm --- Pure python code is copied to here - src - htm --- Python library assembled here - bindings --- C++ extension libraries installed here - regions - tools -``` + bin --- Contains executables for unit tests and examples + include --- Include C++ header files + lib --- Static & Dynamic compiled libraries for htm core found here + - distr ( copy from REPO_DIR/bindings/py/packaging/* by CMake) + / build -- setup.py; setup() puts stuff in here / dist -- setup.py; setup() puts stuff in here | dummy.c | requirements.txt ( copy from REPO_DIR/requirements.txt by setup.py; setup()) | src --setup.py; setup() will look in here for packages, should find htm, tests | htm ( copy from REPO_DIR/py/htm/* by CMake) | advanced | algorithms | bindings ( copy from REPO_DIR/bindings/py/packaging/bindings/* by CMake) | __init__.py ( copy from REPO_DIR/bindings/py/packaging, by CMake) | *.pyd (the extension libraries from CMake build) | check.py Package used regions to build tools Wheel encoders | examples ( copy from REPO_DIR/bindings/py/packaging/examples/* by CMake) \ optimization ( copy form REPO_DIR/py/htm/optimization by CMake) \ README.md ( copy form REPO_DIR/py/htm by CMake) - utils_test.py ``` diff --git a/bindings/py/cpp_src/CMakeLists.txt b/bindings/py/cpp_src/CMakeLists.txt index a8f7d847d3..f56fcbf49f 100644 --- a/bindings/py/cpp_src/CMakeLists.txt +++ b/bindings/py/cpp_src/CMakeLists.txt @@ -102,8 +102,10 @@ source_group("test" FILES ${src_py_test_files}) # setup the distr directory set(distr ${CMAKE_INSTALL_PREFIX}/distr) file(MAKE_DIRECTORY ${distr}) -file(COPY ../packaging/. DESTINATION ${distr} PATTERN *) -file(COPY ${REPOSITORY_DIR}/py/. DESTINATION ${distr}/src PATTERN *) +file(COPY ${REPOSITORY_DIR}/bindings/py/packaging/. DESTINATION ${distr} REGEX ".*__pycache__" EXCLUDE) +#file(COPY ${REPOSITORY_DIR}/bindings/py/tests/. DESTINATION ${distr}/src/htm/tests REGEX ".*__pycache__" EXCLUDE) +file(COPY ${REPOSITORY_DIR}/py/htm/. DESTINATION ${distr}/src/htm REGEX ".*__pycache__" EXCLUDE) +#file(COPY ${REPOSITORY_DIR}/py/tests/. DESTINATION ${distr}/src/htm/tests REGEX ".*__pycache__" EXCLUDE) # determine which version of binding to build if ("${BINDING_BUILD}" STREQUAL "Python2") diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py deleted file mode 100644 index 51fa54bb75..0000000000 --- a/bindings/py/packaging/setup.py +++ /dev/null @@ -1,445 +0,0 @@ -# ---------------------------------------------------------------------- -# HTM Community Edition of NuPIC -# Copyright (C) 2015, Numenta, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero Public License for more details. -# -# You should have received a copy of the GNU Affero Public License -# along with this program. If not, see http://www.gnu.org/licenses. -# ---------------------------------------------------------------------- - -"""This file builds and installs the HTM Core Python bindings.""" - -import glob -import os -import subprocess -import sys -import tempfile -import distutils.dir_util -import json - -from setuptools import Command, find_packages, setup -from setuptools.command.test import test as BaseTestCommand -from distutils.core import Extension - -# NOTE: To debug the python bindings in a debugger, use the procedure -# described here: https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_in_ide.html -# - -# NOTE: CMake usually is able to determine the tool chain based on the platform. -# However, if you would like CMake to use a different generator, (perhaps an -# alternative compiler) you can set the environment variable NC_CMAKE_GENERATOR -# to the generator you wish to use. See CMake docs for generators avaiable. -# -# On Windows, CMake will try to use the newest Visual Studio installed -# on your machine. You many choose an older version as follows: -# set NC_CMAKE_GENERATOR="Visual Studio 15 2017 Win64" -# python setup.py install --user --force -# This script will override the default 32bit bitness such that a 64bit build is created. -# - - -# bindings cannot be compiled in Debug mode, unless a special python library also in -# Debug is used, which is quite unlikely. So for any CMAKE_BUILD_TYPE setting, override -# to Release mode. -build_type = 'Release' - -PY_BINDINGS = os.path.dirname(os.path.realpath(__file__)) -REPO_DIR = os.path.abspath(os.path.join(PY_BINDINGS, os.pardir, os.pardir, os.pardir)) -DISTR_DIR = os.path.join(REPO_DIR, "build", build_type, "distr") -DARWIN_PLATFORM = "darwin" -LINUX_PLATFORM = "linux" -UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM] -WINDOWS_PLATFORMS = ["windows"] - - - -def getExtensionVersion(): - """ - Get version from local file. - """ - with open(os.path.join(REPO_DIR, "VERSION"), "r") as versionFile: - return versionFile.read().strip() - - - -class CleanCommand(Command): - """Command for cleaning up intermediate build files.""" - - description = "Command for cleaning up generated extension files." - user_options = [] - - - def initialize_options(self): - pass - - - def finalize_options(self): - pass - - - def run(self): - platform = getPlatformInfo() - files = getExtensionFileNames(platform) - for f in files: - try: - os.remove(f) - except OSError: - pass - - - -class ConfigureCommand(Command): - """Setup C++ dependencies and call cmake for configuration""" - - description = "Setup C++ dependencies and call cmake for configuration." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - platform = getPlatformInfo() - if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: - os.system("export ARCHFLAGS=\"-arch x86_64\"") - - # Run CMake if extension files are missing. - # CMake also copies all py files into place in DISTR_DIR - configure(platform, build_type) - - - -def fixPath(path): - """ - Ensures paths are correct for linux and windows - """ - path = os.path.abspath(os.path.expanduser(path)) - if path.startswith("\\"): - return "C:" + path - - return path - - - -def findRequirements(platform, fileName="requirements.txt"): - """ - Read the requirements.txt file and parse into requirements for setup's - install_requirements option. - """ - requirementsPath = fixPath(os.path.join(REPO_DIR, fileName)) - return [ - line.strip() - for line in open(requirementsPath).readlines() - if not line.startswith("#") - ] - - - -class TestCommand(BaseTestCommand): - user_options = [("pytest-args=", "a", "Arguments to pass to py.test")] - - - def initialize_options(self): - BaseTestCommand.initialize_options(self) - self.pytest_args = [] # pylint: disable=W0201 - - - def finalize_options(self): - BaseTestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - - def run_tests(self): - import pytest - cwd = os.getcwd() - errno = 0 - # run c++ tests (from python) - cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests") - subprocess.check_call([cpp_tests]) - os.chdir(cwd) - - # run python bindings tests (in /bindings/py/tests/) - try: - os.chdir(os.path.join(REPO_DIR, "bindings", "py", "tests")) - errno = pytest.main(self.pytest_args) - finally: - os.chdir(cwd) - if errno != 0: - sys.exit(errno) - - # python tests (in /py/tests/) - try: - os.chdir(os.path.join(REPO_DIR, "py", "tests")) - errno = pytest.main(self.pytest_args) - finally: - os.chdir(cwd) - sys.exit(errno) - - - -def getPlatformInfo(): - """Identify platform.""" - if "linux" in sys.platform: - platform = "linux" - elif "darwin" in sys.platform: - platform = "darwin" - # win32 - elif sys.platform.startswith("win"): - platform = "windows" - else: - raise Exception("Platform '%s' is unsupported!" % sys.platform) - return platform - - - -def getExtensionFileNames(platform, build_type): - # look for extension libraries in Repository/build/Release/distr/src/htm/bindings - # library filenames: - # htm.core.algorithms.so - # htm.core.engine.so - # htm.core.math.so - # htm.core.encoders.so - # htm.core.sdr.so - # (or on windows x64 with Python3.7:) - # algorithms.cp37-win_amd64.pyd - # engine_internal.cp37-win_amd64.pyd - # math.cp37-win_amd64.pyd - # encoders.cp37-win_amd64.pyd - # sdr.cp37-win_amd64.pyd - if platform in WINDOWS_PLATFORMS: - libExtension = "pyd" - else: - libExtension = "so" - libNames = ("sdr", "encoders", "algorithms", "engine_internal", "math") - libFiles = ["{}.*.{}".format(name, libExtension) for name in libNames] - files = [os.path.join(DISTR_DIR, "src", "htm", "bindings", name) - for name in list(libFiles)] - return files - - -def getExtensionFiles(platform, build_type): - files = getExtensionFileNames(platform, build_type) - for f in files: - if not glob.glob(f): - generateExtensions(platform, build_type) - break - - return files - -def isMSVC_installed(ver): - """ - For windows we need to know the most recent version of Visual Studio that is installed. - This is because the calling arguments for setting x64 is different between 2017 and 2019. - - Run vswhere to get Visual Studio info. (only available in MSVC 2017 and later) - Parse the json and look in displayName for "2017" or "2019" - return true if ver is found. - """ - vswhere = "C:\\Program Files (x86)\\Microsoft Visual Studio\\Installer\\vswhere.exe" - output = subprocess.check_output([vswhere, "-legacy", "-prerelease", "-format", "json"], universal_newlines=True) - data = json.loads(output); - for vs in data: - if 'displayName' in vs and ver in vs['displayName']: return True - return False - - -def generateExtensions(platform, build_type): - """ - This will perform a full Release build with default arguments. - The CMake build will copy everything in the Repository/bindings/py/packaging - directory to the distr directory (Repository/build/Release/distr) - and then create the extension libraries in Repository/build/Release/distr/src/nupic/bindings. - Note: for Windows it will force a X64 build. - """ - cwd = os.getcwd() - scriptsDir = os.path.join(REPO_DIR, "build", "scripts") - try: - if not os.path.isdir(scriptsDir): - os.makedirs(scriptsDir) - os.chdir(scriptsDir) - - # cmake ../.. - configure(platform, build_type) - - # build: make && make install - if platform != "windows": #TODO since cmake 3.12 "-j4" is directly supported (=crossplatform), for now -- passes other options to make - subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type, "--", "-j", "4"]) - else: - subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) - finally: - os.chdir(cwd) - - -def configure(platform, build_type): - """ - Setup C++ dependencies and call cmake for configuration. - """ - cwd = os.getcwd() - - print("Python version: {}\n".format(sys.version)) - from sys import version_info - if version_info > (3, 0): - # Build a Python 3.x library - PY_VER = "-DBINDING_BUILD=Python3" - else: - # Build a Python 2.7 library - PY_VER = "-DBINDING_BUILD=Python2" - if platform == "windows": - raise Exception("Python2 is not supported on Windows.") - - - BUILD_TYPE = "-DCMAKE_BUILD_TYPE="+build_type - - - scriptsDir = os.path.join(REPO_DIR, "build", "scripts") - try: - if not os.path.isdir(scriptsDir): - os.makedirs(scriptsDir) - os.chdir(scriptsDir) - - # Call CMake to setup the cache for the build. - # Normally we would let CMake figure out the generator based on the platform. - # But Visual Studio gets it wrong. By default it uses 32 bit and we support only x64. - # Also Visual Studio 2019 now wants a new argument -A to specify that we want x64. - # Using -A on 2017 causes an error. So we have to manually specify each. - generator = os.environ.get('NC_CMAKE_GENERATOR') - if generator == None: - # The generator is not specified, figure out which to use. - if platform == "windows": - # Check to see if the CMake cache already exists and defines BINDING_BUILD. If it does, skip this step - if not os.path.isfile('CMakeCache.txt') or not 'BINDING_BUILD:STRING=Python3' in open('CMakeCache.txt').read(): - # Note: the calling arguments for MSVC 2017 is not the same as for MSVC 2019 - if isMSVC_installed("2019"): - subprocess.check_call(["cmake", "-G", "Visual Studio 16 2019", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR]) - elif isMSVC_installed("2017"): - subprocess.check_call(["cmake", "-G", "Visual Studio 15 2017 Win64", BUILD_TYPE, PY_VER, REPO_DIR]) - else: - raise Exception("Did not find Microsoft Visual Studio 2017 or 2019.") - #else - # we can skip this step, the cache is already setup and we have the right binding specified. - else: - # For Linux and OSx we can let CMake figure it out. - subprocess.check_call(["cmake",BUILD_TYPE , PY_VER, REPO_DIR]) - - else: - # The generator is specified. - if platform == "windows": - # Check to see if cache already exists. If it does, skip this step - if not os.path.isfile("CMakeCache.txt"): - # Note: the calling arguments for MSVC 2017 is not the same as for MSVC 2019 - if '2019' in generator and isMSVC_installed("2019"): - subprocess.check_call(["cmake", "-G", "Visual Studio 16 2019", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR]) - elif '2017' in generator and isMSVC_installed("2017"): - subprocess.check_call(["cmake", "-G", "Visual Studio 15 2017 Win64", BUILD_TYPE, PY_VER, REPO_DIR]) - else: - raise Exception('Did not find Visual Studio for generator "'+generator+ '".') - else: - subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR]) - - finally: - os.chdir(cwd) - - - -if __name__ == "__main__": - platform = getPlatformInfo() - - if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: - os.system("export ARCHFLAGS=\"-arch x86_64\"") - - # Run CMake if extension files are missing. - # CMake also copies all py files into place in DISTR_DIR - if len(sys.argv) > 1 and sys.argv[1] == "configure": - configure(platform, build_type) - else: #full build - getExtensionFiles(platform, build_type) - - with open(os.path.join(REPO_DIR, "README.md"), "r") as fh: - long_description = fh.read() - - """ - set the default directory to the distr, and package it. - """ - print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n") - os.chdir(DISTR_DIR) - - setup( - # See https://docs.python.org/2/distutils/apiref.html for descriptions of arguments. - # https://docs.python.org/2/distutils/setupscript.html - # https://opensourceforu.com/2010/OS/extending-python-via-shared-libraries - # https://docs.python.org/3/library/ctypes.html - # https://docs.python.org/2/library/imp.html - name="htm.core", - version=getExtensionVersion(), - # This distribution contains platform-specific C++ libraries, but they are not - # built with distutils. So we must create a dummy Extension object so when we - # create a binary file it knows to make it platform-specific. - ext_modules=[Extension('htm.dummy', sources = ['dummy.c'])], - package_dir = {"": "src"}, - packages=find_packages("src"), - namespace_packages=["htm"], - install_requires=findRequirements(platform), - package_data={ - "htm.bindings": ["*.so", "*.pyd"], - "htm.examples": ["*.csv"], - }, - #install extras by `pip install htm.core[examples]` - extras_require={'scikit-image>0.15.0':'examples', - 'sklearn':'examples', - 'matplotlib':'examples', - 'PIL':'examples', - 'scipy':'examples' - }, - zip_safe=False, - cmdclass={ - "clean": CleanCommand, - "test": TestCommand, - "configure": ConfigureCommand, - }, - author="Numenta & HTM Community", - author_email="help@numenta.org", - url="https://github.com/htm-community/htm.core", - description = "HTM Community Edition of Numenta's Platform for Intelligent Computing (NuPIC) htm.core", - long_description = long_description, - long_description_content_type="text/markdown", - license = "GNU Affero General Public License v3 or later (AGPLv3+)", - classifiers=[ - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Operating System :: POSIX :: BSD", - "Operating System :: Microsoft :: Windows", - "Operating System :: OS Independent", - # It has to be "5 - Production/Stable" or else pypi rejects it! - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Science/Research", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Natural Language :: English", - "Programming Language :: C++", - "Programming Language :: Python" - ], - entry_points = { - "console_scripts": [ - "htm-bindings-check = htm.bindings.check:checkMain", - ], - }, - ) - print("\nbindings/py/setup.py: Setup complete.\n") - diff --git a/bindings/py/tests/__init__.py b/bindings/py/tests/__init__.py old mode 100755 new mode 100644 diff --git a/bindings/py/tests/algorithms/temporal_memory_test.py b/bindings/py/tests/algorithms/temporal_memory_test.py old mode 100755 new mode 100644 diff --git a/bindings/py/tests/check_test.py b/bindings/py/tests/check_test.py old mode 100755 new mode 100644 diff --git a/bindings/py/tests/nupic_random_test.py b/bindings/py/tests/nupic_random_test.py old mode 100755 new mode 100644 diff --git a/bindings/py/tests/regions/network_test.py b/bindings/py/tests/regions/network_test.py old mode 100755 new mode 100644 diff --git a/bindings/py/tests/regions/pyregion_test.py b/bindings/py/tests/regions/pyregion_test.py old mode 100755 new mode 100644 diff --git a/bindings/py/tests/sparse_link_test.py b/bindings/py/tests/sparse_link_test.py old mode 100755 new mode 100644 diff --git a/external/YamlCppLib.cmake b/external/YamlCppLib.cmake index d1cb12e486..fa7d7e19bb 100644 --- a/external/YamlCppLib.cmake +++ b/external/YamlCppLib.cmake @@ -20,7 +20,6 @@ message(STATUS "${REPOSITORY_DIR}/build/ThirdParty/share/yaml-cpp.zip") if(EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/yaml-cpp.zip") set(URL "${REPOSITORY_DIR}/build/ThirdParty/share/yaml-cpp.zip") else() - #set(URL https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.2.tar.gz) # There seems to be something wrong with the 0.6.2 distribution. Use the v0.6.3. set(URL https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.3.zip) endif() diff --git a/setup.py b/setup.py index 1cef637e0a..9b1e0055bb 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# ---------------------------------------------------------------------- +# ---------------------------------------------------------------------- # HTM Community Edition of NuPIC # Copyright (C) 2015, Numenta, Inc. # @@ -14,40 +14,433 @@ # You should have received a copy of the GNU Affero Public License # along with this program. If not, see http://www.gnu.org/licenses. # ---------------------------------------------------------------------- -""" -Based on: http://stackoverflow.com/questions/5317672/pip-not-finding-setup-file -""" +"""This file builds and installs the HTM Core Python bindings.""" + +import glob import os +import subprocess +import sys +import tempfile +import distutils.dir_util +import json + +from setuptools import Command, find_packages, setup +from setuptools.command.test import test as BaseTestCommand +from distutils.core import Extension + +# NOTE: To debug the python bindings in a debugger, use the procedure +# described here: https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_in_ide.html +# + +# NOTE: CMake usually is able to determine the tool chain based on the platform. +# However, if you would like CMake to use a different generator, (perhaps an +# alternative compiler) you can set the environment variable NC_CMAKE_GENERATOR +# to the generator you wish to use. See CMake docs for generators avaiable. +# +# On Windows, CMake will try to use the newest Visual Studio installed +# on your machine. You many choose an older version as follows: +# set NC_CMAKE_GENERATOR="Visual Studio 15 2017 Win64" +# python setup.py install --user --force +# This script will override the default 32bit bitness such that a 64bit build is created. +# -from setuptools.command import egg_info + +# bindings cannot be compiled in Debug mode, unless a special python library also in +# Debug is used, which is quite unlikely. So for any CMAKE_BUILD_TYPE setting, override +# to Release mode. +build_type = 'Release' REPO_DIR = os.path.dirname(os.path.realpath(__file__)) -filename = os.path.basename(__file__) +PY_BINDINGS = os.path.join(REPO_DIR, "bindings", "py") +DISTR_DIR = os.path.join(REPO_DIR, "build", build_type, "distr") +DARWIN_PLATFORM = "darwin" +LINUX_PLATFORM = "linux" +UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM] +WINDOWS_PLATFORMS = ["windows"] + + + +def getExtensionVersion(): + """ + Get version from local file. + """ + with open(os.path.join(REPO_DIR, "VERSION"), "r") as versionFile: + return versionFile.read().strip() + + + +class CleanCommand(Command): + """Command for cleaning up intermediate build files.""" + + description = "Command for cleaning up generated extension files." + user_options = [] + + + def initialize_options(self): + pass + + + def finalize_options(self): + pass + + + def run(self): + platform = getPlatformInfo() + files = getExtensionFileNames(platform) + for f in files: + try: + os.remove(f) + except OSError: + pass + + + +class ConfigureCommand(Command): + """Setup C++ dependencies and call cmake for configuration""" + + description = "Setup C++ dependencies and call cmake for configuration." + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + platform = getPlatformInfo() + if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: + os.system("export ARCHFLAGS=\"-arch x86_64\"") + + # Run CMake if extension files are missing. + # CMake also copies all py files into place in DISTR_DIR + configure(platform, build_type) + + + +def fixPath(path): + """ + Ensures paths are correct for linux and windows + """ + path = os.path.abspath(os.path.expanduser(path)) + if path.startswith("\\"): + return "C:" + path + + return path + + + +def findRequirements(platform, fileName="requirements.txt"): + """ + Read the requirements.txt file and parse into requirements for setup's + install_requirements option. + """ + requirementsPath = fixPath(os.path.join(REPO_DIR, fileName)) + return [ + line.strip() + for line in open(requirementsPath).readlines() + if not line.startswith("#") + ] + + + +class TestCommand(BaseTestCommand): + user_options = [("pytest-args=", "a", "Arguments to pass to py.test")] + + + def initialize_options(self): + BaseTestCommand.initialize_options(self) + self.pytest_args = [] # pylint: disable=W0201 + + + def finalize_options(self): + BaseTestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + + def run_tests(self): + import pytest + cwd = os.getcwd() + errno = 0 + # run c++ tests (from python) + cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests") + subprocess.check_call([cpp_tests]) + os.chdir(cwd) + + + # run python bindings tests (in /bindings/py/tests/) + try: + os.chdir(os.path.join(REPO_DIR, "bindings", "py","tests")) + errno = pytest.main(self.pytest_args) + finally: + os.chdir(cwd) + if errno != 0: + sys.exit(errno) + + # python tests (in /py/tests/) + try: + os.chdir(os.path.join(REPO_DIR, "py", "tests")) + errno = pytest.main(self.pytest_args) + finally: + os.chdir(cwd) + sys.exit(errno) + + + +def getPlatformInfo(): + """Identify platform.""" + if "linux" in sys.platform: + platform = "linux" + elif "darwin" in sys.platform: + platform = "darwin" + # win32 + elif sys.platform.startswith("win"): + platform = "windows" + else: + raise Exception("Platform '%s' is unsupported!" % sys.platform) + return platform + + + +def getExtensionFileNames(platform, build_type): + # look for extension libraries in Repository/build/Release/distr/src/htm/bindings + # library filenames: + # htm.core.algorithms.so + # htm.core.engine.so + # htm.core.math.so + # htm.core.encoders.so + # htm.core.sdr.so + # (or on windows x64 with Python3.7:) + # algorithms.cp37-win_amd64.pyd + # engine_internal.cp37-win_amd64.pyd + # math.cp37-win_amd64.pyd + # encoders.cp37-win_amd64.pyd + # sdr.cp37-win_amd64.pyd + if platform in WINDOWS_PLATFORMS: + libExtension = "pyd" + else: + libExtension = "so" + libNames = ("sdr", "encoders", "algorithms", "engine_internal", "math") + libFiles = ["{}.*.{}".format(name, libExtension) for name in libNames] + files = [os.path.join(DISTR_DIR, "src", "htm", "bindings", name) + for name in list(libFiles)] + return files + + +def getExtensionFiles(platform, build_type): + files = getExtensionFileNames(platform, build_type) + for f in files: + if not glob.glob(f): + generateExtensions(platform, build_type) + break + + return files + +def isMSVC_installed(ver): + """ + For windows we need to know the most recent version of Visual Studio that is installed. + This is because the calling arguments for setting x64 is different between 2017 and 2019. + + Run vswhere to get Visual Studio info. (only available in MSVC 2017 and later) + Parse the json and look in displayName for "2017" or "2019" + return true if ver is found. + """ + vswhere = "C:\\Program Files (x86)\\Microsoft Visual Studio\\Installer\\vswhere.exe" + output = subprocess.check_output([vswhere, "-legacy", "-prerelease", "-format", "json"], universal_newlines=True) + data = json.loads(output); + for vs in data: + if 'displayName' in vs and ver in vs['displayName']: return True + return False + + +def generateExtensions(platform, build_type): + """ + This will perform a full Release build with default arguments. + The CMake build will copy everything in the Repository/bindings/py/packaging + directory to the distr directory (Repository/build/Release/distr) + and then create the extension libraries in Repository/build/Release/distr/src/nupic/bindings. + Note: for Windows it will force a X64 build. + """ + cwd = os.getcwd() + scriptsDir = os.path.join(REPO_DIR, "build", "scripts") + try: + if not os.path.isdir(scriptsDir): + os.makedirs(scriptsDir) + os.chdir(scriptsDir) + + # cmake ../.. + configure(platform, build_type) + + # build: make && make install + if platform != "windows": #TODO since cmake 3.12 "-j4" is directly supported (=crossplatform), for now -- passes other options to make + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type, "--", "-j", "4"]) + else: + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) + finally: + os.chdir(cwd) + + +def configure(platform, build_type): + """ + Setup C++ dependencies and call cmake for configuration. + """ + cwd = os.getcwd() + + print("Python version: {}\n".format(sys.version)) + from sys import version_info + if version_info > (3, 0): + # Build a Python 3.x library + PY_VER = "-DBINDING_BUILD=Python3" + else: + # Build a Python 2.7 library + PY_VER = "-DBINDING_BUILD=Python2" + if platform == "windows": + raise Exception("Python2 is not supported on Windows.") + + + BUILD_TYPE = "-DCMAKE_BUILD_TYPE="+build_type + + + scriptsDir = os.path.join(REPO_DIR, "build", "scripts") + try: + if not os.path.isdir(scriptsDir): + os.makedirs(scriptsDir) + os.chdir(scriptsDir) + + # Call CMake to setup the cache for the build. + # Normally we would let CMake figure out the generator based on the platform. + # But Visual Studio gets it wrong. By default it uses 32 bit and we support only x64. + # Also Visual Studio 2019 now wants a new argument -A to specify that we want x64. + # Using -A on 2017 causes an error. So we have to manually specify each. + generator = os.environ.get('NC_CMAKE_GENERATOR') + if generator == None: + # The generator is not specified, figure out which to use. + if platform == "windows": + # Check to see if the CMake cache already exists and defines BINDING_BUILD. If it does, skip this step + if not os.path.isfile('CMakeCache.txt') or not 'BINDING_BUILD:STRING=Python3' in open('CMakeCache.txt').read(): + # Note: the calling arguments for MSVC 2017 is not the same as for MSVC 2019 + if isMSVC_installed("2019"): + subprocess.check_call(["cmake", "-G", "Visual Studio 16 2019", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR]) + elif isMSVC_installed("2017"): + subprocess.check_call(["cmake", "-G", "Visual Studio 15 2017 Win64", BUILD_TYPE, PY_VER, REPO_DIR]) + else: + raise Exception("Did not find Microsoft Visual Studio 2017 or 2019.") + #else + # we can skip this step, the cache is already setup and we have the right binding specified. + else: + # For Linux and OSx we can let CMake figure it out. + subprocess.check_call(["cmake",BUILD_TYPE , PY_VER, REPO_DIR]) + + else: + # The generator is specified. + if platform == "windows": + # Check to see if cache already exists. If it does, skip this step + if not os.path.isfile("CMakeCache.txt"): + # Note: the calling arguments for MSVC 2017 is not the same as for MSVC 2019 + if '2019' in generator and isMSVC_installed("2019"): + subprocess.check_call(["cmake", "-G", "Visual Studio 16 2019", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR]) + elif '2017' in generator and isMSVC_installed("2017"): + subprocess.check_call(["cmake", "-G", "Visual Studio 15 2017 Win64", BUILD_TYPE, PY_VER, REPO_DIR]) + else: + raise Exception('Did not find Visual Studio for generator "'+generator+ '".') + else: + subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR]) + + finally: + os.chdir(cwd) -os.chdir(os.path.join(REPO_DIR, "bindings","py","packaging")) -setupdir = os.getcwd() -egginfo = "pip-egg-info" -__file__ = os.path.join(setupdir, filename) +if __name__ == "__main__": + platform = getPlatformInfo() -def replacement_run(self): - print("setup.py::replacement_run()\n") - self.mkpath(self.egg_info) + if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: + os.system("export ARCHFLAGS=\"-arch x86_64\"") - installer = self.distribution.fetch_build_egg + # Run CMake if extension files are missing. + # CMake also copies all py files into place in DISTR_DIR + if len(sys.argv) > 1 and sys.argv[1] == "configure": + configure(platform, build_type) + else: #full build + getExtensionFiles(platform, build_type) - for ep in egg_info.iter_entry_points('egg_info.writers'): - # require=False is the change we're making from pip - writer = ep.load(require=False) + with open(os.path.join(REPO_DIR, "README.md"), "r") as fh: + long_description = fh.read() - if writer: - writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name)) + """ + set the default directory to the distr, and package it. + """ + print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n") + os.chdir(DISTR_DIR) - self.find_sources() + setup( + # See https://docs.python.org/2/distutils/apiref.html for descriptions of arguments. + # https://docs.python.org/2/distutils/setupscript.html + # https://opensourceforu.com/2010/OS/extending-python-via-shared-libraries + # https://docs.python.org/3/library/ctypes.html + # https://docs.python.org/2/library/imp.html + name="htm.core", + version=getExtensionVersion(), + # This distribution contains platform-specific C++ libraries, but they are not + # built with distutils. So we must create a dummy Extension object so when we + # create a binary file it knows to make it platform-specific. + ext_modules=[Extension('htm.dummy', sources = ['dummy.c'])], + package_dir = {"": "src"}, + packages=find_packages("src"), + namespace_packages=["htm"], + install_requires=findRequirements(platform), + package_data={ + "htm.bindings": ["*.so", "*.pyd"], + "htm.examples": ["*.csv"], + }, + #install extras by `pip install htm.core[examples]` + extras_require={'scikit-image>0.15.0':'examples', + 'sklearn':'examples', + 'matplotlib':'examples', + 'PIL':'examples', + 'scipy':'examples' + }, + zip_safe=False, + cmdclass={ + "clean": CleanCommand, + "test": TestCommand, + "configure": ConfigureCommand, + }, + author="Numenta & HTM Community", + author_email="help@numenta.org", + url="https://github.com/htm-community/htm.core", + description = "HTM Community Edition of Numenta's Platform for Intelligent Computing (NuPIC) htm.core", + long_description = long_description, + long_description_content_type="text/markdown", + license = "GNU Affero General Public License v3 or later (AGPLv3+)", + classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Operating System :: POSIX :: BSD", + "Operating System :: Microsoft :: Windows", + "Operating System :: OS Independent", + # It has to be "5 - Production/Stable" or else pypi rejects it! + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Natural Language :: English", + "Programming Language :: C++", + "Programming Language :: Python" + ], + entry_points = { + "console_scripts": [ + "htm-bindings-check = htm.bindings.check:checkMain", + ], + }, + ) + print("\nbindings/py/setup.py: Setup complete.\n") -egg_info.egg_info.run = replacement_run -print("setup.py: Calling {}\n".format(__file__)) -import runpy -runpy.run_path(__file__, run_name=__name__)