setup.py

# ----------------------------------------------------------------------
# HTM Community Edition of NuPIC
# Copyright (C) 2015, Numenta, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
# ----------------------------------------------------------------------

"""This file builds and installs the HTM Core Python bindings."""
 
import glob
import os
import subprocess
import sys
import json

from setuptools import Command, find_packages, setup
from setuptools.command.test import test as BaseTestCommand
# see https://stackoverflow.com/questions/44323474/distutils-core-vs-setuptools-with-c-extension
from setuptools import Extension
from pathlib import Path
from sys import version_info
from packaging import version

# NOTE:  To debug the python bindings in a debugger, use the procedure
#        described here: https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_in_ide.html
#

# NOTE:  CMake usually is able to determine the tool chain based on the platform.
#        However, if you would like CMake to use a different generator, (perhaps an 
#        alternative compiler) you can set the environment variable NC_CMAKE_GENERATOR
#        to the generator you wish to use.  See CMake docs for generators avaiable.
#         
#         On Windows, CMake will try to use the newest Visual Studio installed
#         on your machine. You many choose an older version as follows:
#            set NC_CMAKE_GENERATOR="Visual Studio 15 2017 Win64"
#            python setup.py install --user --force
#         This script will override the default 32bit bitness such that a 64bit build is created.
#


# bindings cannot be compiled in Debug mode, unless a special python library also in
# Debug is used, which is quite unlikely. So for any CMAKE_BUILD_TYPE setting, override 
# to Release mode. 
build_type = 'Release'

REPO_DIR = os.path.dirname(os.path.realpath(__file__))
PY_BINDINGS = os.path.join(REPO_DIR, "bindings", "py")
DISTR_DIR = os.path.join(REPO_DIR, "build", build_type, "distr")
DARWIN_PLATFORM = "darwin"
LINUX_PLATFORM = "linux"
UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM]
WINDOWS_PLATFORMS = ["windows"]



def getExtensionVersion():
  """
  Get version from local file.
  """
  with open(os.path.join(REPO_DIR, "VERSION"), "r") as versionFile:
    return versionFile.read().strip()



class CleanCommand(Command):
  """Command for cleaning up intermediate build files."""

  description = "Command for cleaning up generated extension files."
  user_options = []


  def initialize_options(self):
    pass


  def finalize_options(self):
    pass


  def run(self):
    platform = getPlatformInfo()
    files = getExtensionFileNames(platform)
    for f in files:
      try:
        os.remove(f)
      except OSError:
        pass



class ConfigureCommand(Command):
  """Setup C++ dependencies and call cmake for configuration"""

  description = "Setup C++ dependencies and call cmake for configuration."
  user_options = []

  def initialize_options(self):
    pass

  def finalize_options(self):
    pass

  def run(self):
    platform = getPlatformInfo()
    if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ:
      os.system("export ARCHFLAGS=\"-arch x86_64\"")

    # Run CMake if extension files are missing.
    # CMake also copies all py files into place in DISTR_DIR
    configure(platform, build_type)



def fixPath(path):
  """
  Ensures paths are correct for linux and windows
  """
  path = os.path.abspath(os.path.expanduser(path))
  if path.startswith("\\"):
    return "C:" + path

  return path



def findRequirements(platform, fileName="requirements.txt"):
  """
  Read the requirements.txt file and parse into requirements for setup's
  install_requirements option.
  """
  requirementsPath = fixPath(os.path.join(REPO_DIR, fileName))
  return [
    line.strip()
    for line in open(requirementsPath).readlines()
    if not line.startswith("#") 
  ]



class TestCommand(BaseTestCommand):
  user_options = [("pytest-args=", "a", "Arguments to pass to py.test")]


  def initialize_options(self):
    BaseTestCommand.initialize_options(self)
    self.pytest_args = [] # pylint: disable=W0201


  def finalize_options(self):
    BaseTestCommand.finalize_options(self)
    self.test_args = []
    self.test_suite = True


  def run_tests(self):
    import pytest
    cwd = os.getcwd()
    errno = 0
    os.chdir(REPO_DIR)
    # run c++ tests (from python)
    cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests")
    subprocess.check_call([cpp_tests])
    os.chdir(cwd)
    

    # run python bindings tests (in /bindings/py/tests/)
    try:
      os.chdir(os.path.join(REPO_DIR, "bindings", "py","tests"))
      errno = pytest.main(self.pytest_args)
    finally:
      os.chdir(cwd)
    if errno != 0:
      sys.exit(errno)
    
    # python tests (in /py/tests/)
    try:
      os.chdir(os.path.join(REPO_DIR, "py", "tests"))
      errno = pytest.main(self.pytest_args)
    finally:
      os.chdir(cwd)
    sys.exit(errno)



def getPlatformInfo():
  """Identify platform."""
  if "linux" in sys.platform:
    platform = "linux"
  elif "darwin" in sys.platform:
    platform = "darwin"
  # win32
  elif sys.platform.startswith("win"):
    platform = "windows"
  else:
    raise Exception("Platform '%s' is unsupported!" % sys.platform)
  return platform



def getExtensionFileNames(platform, build_type):
  # look for extension libraries in Repository/build/Release/distr/src/htm/bindings
  # library filenames:  
  #     htm.core.algorithms.so
  #     htm.core.engine.so
  #     htm.core.math.so
  #     htm.core.encoders.so
  #     htm.core.sdr.so
  # (or on windows x64 with Python3.7:)
  #     algorithms.cp37-win_amd64.pyd
  #     engine_internal.cp37-win_amd64.pyd
  #     math.cp37-win_amd64.pyd
  #     encoders.cp37-win_amd64.pyd
  #     sdr.cp37-win_amd64.pyd
  if platform in WINDOWS_PLATFORMS:
    libExtension = "pyd"
  else:
    libExtension = "so"
  libNames = ("sdr", "encoders", "algorithms", "engine_internal", "math")
  libFiles = ["{}.*.{}".format(name, libExtension) for name in libNames]
  files = [os.path.join(DISTR_DIR, "src", "htm", "bindings", name)
           for name in list(libFiles)]
  return files


def getExtensionFiles(platform, build_type):
  files = getExtensionFileNames(platform, build_type)
  for f in files:
    if not glob.glob(f):
      generateExtensions(platform, build_type)
      break

  return files
  
def isMSVC_installed(ver):
  """
  For windows we need to know the most recent version of Visual Studio that is installed.
  This is because the calling arguments for Visual Studio is different between the versions.
  
  Run vswhere to get Visual Studio info.  (only available in MSVC 2017 and later)
  Parse the json and look in displayName containing "2017", "2019" or 2022
  return true if ver is found.
  """
  vswhere = "C:\\Program Files (x86)\\Microsoft Visual Studio\\Installer\\vswhere.exe"
  if not os.path.isfile(vswhere):
    raise Exception("Visual Studio not installed.")
  output = subprocess.check_output([vswhere, "-products", "*", "-legacy", "-prerelease", "-format", "json"], universal_newlines=True)
  data = json.loads(output);
  for vs in data:
    if 'displayName' in vs and ver in vs['displayName']: return True
  return False


def getCMakeVersion():
  """
  Get the version from the currently installed CMake program.
  This should work on Windows, OSx and Linux.
  Return the result as an instance of the object packaging.Version.
  These can be compared with >, >=, etc.
  See https://stackoverflow.com/questions/11887762/how-do-i-compare-version-numbers-in-python
  """
  result = subprocess.run(["cmake", "--version"], capture_output=True, text=True)
  if result.stderr=="":
    str = result.stdout
    str = str.split("\n",1)[0].split()[2]
    ver = version.parse(str)
    return ver
  return False
  

def generateExtensions(platform, build_type):
  """
  This will perform a full Release build with default arguments.
  The CMake build will copy everything in the Repository/bindings/py/packaging 
  directory to the distr directory (Repository/build/Release/distr)
  and then create the extension libraries in Repository/build/Release/distr/src/nupic/bindings.
  Note: for Windows it will force a X64 build.
  """
  
  # Make sure that .py code gets copied during any build so it is included in .whl
  if version_info >= (3, 4):
    Path(os.path.join(PY_BINDINGS, 'cpp_src', 'CMakeLists.txt')).touch()
  
  cwd = os.getcwd()
  scriptsDir = os.path.join(REPO_DIR, "build", "scripts")
  try:
    if not os.path.isdir(scriptsDir):
      os.makedirs(scriptsDir)
    os.chdir(scriptsDir)

    # cmake ../..
    configure(platform, build_type)

    # build: make && make install
    if platform != "windows": #TODO since cmake 3.12 "-j4" is directly supported (=crossplatform), for now -- passes other options to make
      subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type, "--", "-j", "4"]) 
    else:
      subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type])
  finally:
    os.chdir(cwd)


def configure(platform, build_type):
  """
  Setup C++ dependencies and call cmake for configuration.
  """
  cwd = os.getcwd()
  
  print("Python version: {}\n".format(sys.version))
  if version_info > (3, 0):
    # Build a Python 3.x library
    PY_VER = "-DBINDING_BUILD=Python3"
  else:
    # Build a Python 2.7 library
    PY_VER = "-DBINDING_BUILD=Python2"
    if platform == "windows":
        raise Exception("Python2 is not supported on Windows.")
        

  BUILD_TYPE = "-DCMAKE_BUILD_TYPE="+build_type


  scriptsDir = os.path.join(REPO_DIR, "build", "scripts")
  try:
    if not os.path.isdir(scriptsDir):
      os.makedirs(scriptsDir)
    os.chdir(scriptsDir)
    
    # Make sure we have CMake installed
    cmake_ver = getCMakeVersion();
    if cmake_ver == False:
      raise Exception("CMake is not found.");
    
    # Call CMake to setup the cache for the build.
    # Normally we would let CMake figure out the generator based on the platform.
    # But Visual Studio gets it wrong.  By default it uses 32 bit and we support only x64.  
    # Also Visual Studio 2019 now wants a new argument -A to specify that we want x64.
    # Using -A on 2017 causes an error.  So we have to manually specify each.
    generator = os.environ.get('NC_CMAKE_GENERATOR')
    if generator == None:
      # The generator is not specified, figure out which to use.
      if platform == "windows":
        # Check to see if the CMake cache already exists and defines BINDING_BUILD.  If it does, skip this step
        if not os.path.isfile('CMakeCache.txt') or not 'BINDING_BUILD:STRING=Python3' in open('CMakeCache.txt').read():
          if isMSVC_installed("2022"):
            if cmake_ver < version.parse("3.21"):
               raise Exception("Microsoft Visual Studio 2022 requires CMake 3.21 or greater.")
            subprocess.check_call(["cmake", "-G", "Visual Studio 17 2022", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR])
          elif isMSVC_installed("2019"):
            if cmake_ver < version.parse("3.14"):
               raise Exception("Microsoft Visual Studio 2019 requires CMake 3.14 or greater")
            subprocess.check_call(["cmake", "-G", "Visual Studio 16 2019", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR])
          elif isMSVC_installed("2017"):
            if cmake_ver < version.parse("3.7"):
               raise Exception("Microsoft Visual Studio 2017 requires CMake 3.7 or greater")
            # Note: the calling arguments for MSVC 2017 is not the same as for MSVC 2019
            subprocess.check_call(["cmake", "-G", "Visual Studio 15 2017 Win64", BUILD_TYPE, PY_VER, REPO_DIR])
          else:
            raise Exception("Did not find Microsoft Visual Studio 2017, 2019, or 2022.")
        #else 
        #   we can skip this step, the cache is already setup and we have the right binding specified.
      else:
        # For Linux and OSx we can let CMake figure it out.
        subprocess.check_call(["cmake",BUILD_TYPE , PY_VER, REPO_DIR])
        
    else:
      # The generator is specified.
      if platform == "windows":
        # Check to see if cache already exists.  If it does, skip this step
        if not os.path.isfile("CMakeCache.txt"):
          if '2022' in generator and isMSVC_installed("2022"):
            if cmake_ver < version.parse("3.21"):
               raise Exception("Microsoft Visual Studio 2022 requires CMake 3.21 or greater.")
            subprocess.check_call(["cmake", "-G", "Visual Studio 17 2022", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR])
          # Note: the calling arguments for MSVC 2017 is not the same as for MSVC 2019
          elif '2019' in generator and isMSVC_installed("2019"):
            if cmake_ver < version.parse("3.14"):
               raise Exception("Microsoft Visual Studio 2019 requires CMake 3.14 or greater")
            subprocess.check_call(["cmake", "-G", "Visual Studio 16 2019", "-A", "x64", BUILD_TYPE, PY_VER, REPO_DIR])
          elif '2017' in generator and isMSVC_installed("2017"):
            if cmake_ver < version.parse("3.7"):
               raise Exception("Microsoft Visual Studio 2017 requires CMake 3.7 or greater")
            subprocess.check_call(["cmake", "-G", "Visual Studio 15 2017 Win64", BUILD_TYPE, PY_VER, REPO_DIR])
          else:
            raise Exception('Did not find Visual Studio for generator "'+generator+ '".')
      else:
        subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR])
        
  finally:
    os.chdir(cwd)



if __name__ == "__main__":
  platform = getPlatformInfo()

  if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ:
    os.system("export ARCHFLAGS=\"-arch x86_64\"")

  # Run CMake if extension files are missing.
  # CMake also copies all py files into place in DISTR_DIR
  if len(sys.argv) > 1 and sys.argv[1] == "configure":
    configure(platform, build_type)
  else: #full build
    getExtensionFiles(platform, build_type)

  with open(os.path.join(REPO_DIR, "README.md"), "r") as fh:
    long_description = fh.read()

  """
  set the default directory to the distr, and package it.
  """
  print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n")
  os.chdir(DISTR_DIR)

  setup(
    # See https://setuptools.readthedocs.io/en/latest/setuptools.html
    #     https://opensourceforu.com/2010/OS/extending-python-via-shared-libraries
    #     https://docs.python.org/3/library/ctypes.html
    #     https://docs.python.org/2/library/imp.html
    name="htm.core",
    version=getExtensionVersion(),
    # This distribution contains platform-specific C++ libraries, but they are not
    # built with distutils. So we must create a dummy Extension object so when we
    # create a binary file it knows to make it platform-specific.
    ext_modules=[Extension('htm.dummy', sources = ['dummy.c'])],
    package_dir = {"": "src"},
    packages=find_packages("src"),
    namespace_packages=["htm"],
    install_requires=findRequirements(platform),
    package_data={
        "htm.bindings": ["*.so", "*.pyd"],
        "htm.examples": ["*.csv"],
    },
    #install extras by `pip install htm.core[examples]`
    extras_require={'examples':['scikit-image>0.15.0',
                    'sklearn',
                    'matplotlib',
                    'pillow',
                    'requests',
                    'scipy']
                   },
    zip_safe=False,
    cmdclass={
      "clean": CleanCommand,
      "test": TestCommand,
      "configure": ConfigureCommand,
    },
    author="Numenta & HTM Community",
    author_email="help@numenta.org",
    url="https://github.com/htm-community/htm.core",
    description = "HTM Community Edition of Numenta's Platform for Intelligent Computing (NuPIC) htm.core",
    long_description = long_description,
    long_description_content_type="text/markdown",
    license = "GNU Affero General Public License v3 or later (AGPLv3+)",
    classifiers=[
      "Programming Language :: Python",
      "Programming Language :: Python :: 2",
      "Programming Language :: Python :: 3",
      "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
      "Operating System :: MacOS :: MacOS X",
      "Operating System :: POSIX :: Linux",
      "Operating System :: POSIX :: BSD",
      "Operating System :: Microsoft :: Windows",
      "Operating System :: OS Independent",
      # It has to be "5 - Production/Stable" or else pypi rejects it!
      "Development Status :: 5 - Production/Stable",
      "Environment :: Console",
      "Intended Audience :: Science/Research",
      "Intended Audience :: Developers",
      "Intended Audience :: Education",
      "Topic :: Scientific/Engineering :: Artificial Intelligence",
      "Natural Language :: English",
      "Programming Language :: C++",
      "Programming Language :: Python"
    ],
    entry_points = {
      "console_scripts": [
        "htm-bindings-check = htm.bindings.check:checkMain",
      ],
    },
  )
  print("\nbindings/py/setup.py: Setup complete.\n")