Make python package buildable by cibuildwheel tool

Make hyperonc and hyperonpy two separate CMake projects. Re-write
setup.py to build hyperonpy using CMake. hyperonc is expected to be
installed before hyperonpy is built. Add different tricks to make it
buildable under cibuildwheel containers:
- add install-hyperonc.sh which builds and install hyperonc
- pass repo and commit into install-hyperon.sh to install correct
  versions of hyperonc
- allow build only static hyperonc because shared library cannot be
  built under musllinux-x86_64
- ignore mulllinux-i686 where Rust compiler cannot be installed
Add root project to build both hyperonc and hyperonpy.

CMakeLists.txt

@@ -3,15 +3,37 @@ cmake_minimum_required(VERSION 3.10.2)
 
 project(hyperon)
 
-# Fix behavior of CMAKE_CXX_STANDARD when targeting macOS.
-if (POLICY CMP0025)
-    cmake_policy(SET CMP0025 NEW)
-endif ()
-set(CMAKE_CXX_STANDARD 11)
+include(ExternalProject)
 
-enable_testing()
-add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure)
+set(HYPERONC_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/hyperonc-prefix")
+message(STATUS "HYPERONC_INSTALL_PREFIX = ${HYPERONC_INSTALL_PREFIX}")
+
+ExternalProject_Add(
+    hyperonc
+    BUILD_ALWAYS 1
+    PREFIX "${HYPERONC_INSTALL_PREFIX}"
+    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/c"
+    CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${HYPERONC_INSTALL_PREFIX}"
+)
+
+ExternalProject_Get_Property(hyperonc BINARY_DIR)
+set(HYPERONC_BINARY_DIR "${BINARY_DIR}")
+message(STATUS "HYPERONC_BINARY_DIR = ${HYPERONC_BINARY_DIR}")
 
-add_subdirectory(c)
-add_subdirectory(python)
+ExternalProject_Add(
+    hyperonpy
+    BUILD_ALWAYS 1
+    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/python"
+    DEPENDS hyperonc
+    CMAKE_ARGS "-DHYPERONC_INSTALL_PREFIX=${HYPERONC_INSTALL_PREFIX}" "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR}/python"
+    INSTALL_COMMAND cmake -E echo "Skipping install step."
+)
 
+ExternalProject_Get_Property(hyperonpy BINARY_DIR)
+set(HYPERONPY_BINARY_DIR "${BINARY_DIR}")
+message(STATUS "HYPERONPY_BINARY_DIR = ${HYPERONPY_BINARY_DIR}")
+
+enable_testing()
+add_custom_target(check
+    COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --test-dir "${HYPERONC_BINARY_DIR}"
+    COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --test-dir "${HYPERONPY_BINARY_DIR}")

c/CMakeLists.txt

@@ -1,5 +1,13 @@
+# minimum version required by $<IF..> generator expression
+cmake_minimum_required(VERSION 3.10.2)
+project(hyperonc)
+
+enable_testing()
+option(BUILD_SHARED_LIBS "Build shared library" ON)
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure)
+
 execute_process(
-    COMMAND conan install --build missing ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND conan install --build -- ${CMAKE_CURRENT_SOURCE_DIR}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 include(${CMAKE_CURRENT_BINARY_DIR}/conanbuildinfo.cmake)
 conan_basic_setup(TARGETS)
@@ -19,8 +27,6 @@ set(RUST_BUILD_SUFFIX $<IF:${IS_RELEASE_BUILD},release,debug>)
 add_custom_target(show-build-type-vars COMMAND ${CMAKE_COMMAND} -E echo
     "CMAKE_BUILD_TYPE:${BUILD_TYPE} IS_RELEASE_BUILD=${IS_RELEASE_BUILD} RUST_BUILD_SUFFIX=${RUST_BUILD_SUFFIX}")
 
-set(HYPERONC_SHARED_LIB_FILE ${CMAKE_SHARED_LIBRARY_PREFIX}hyperonc${CMAKE_SHARED_LIBRARY_SUFFIX})
-set(HYPERONC_SHARED_LIB_PATH ${HYPERONC_TARGET_DIR}/${HYPERONC_SHARED_LIB_FILE})
 set(HYPERONC_STATIC_LIB_FILE ${CMAKE_STATIC_LIBRARY_PREFIX}hyperonc${CMAKE_STATIC_LIBRARY_SUFFIX})
 set(HYPERONC_STATIC_LIB_PATH ${HYPERONC_TARGET_DIR}/${HYPERONC_STATIC_LIB_FILE})
 set(HYPERONC_INCLUDE_DIR ${HYPERONC_TARGET_DIR}/hyperon)
@@ -31,9 +37,6 @@ add_custom_target(build-hyperonc ALL
         --target-dir ${HYPERONC_TARGET_DIR}
 
     COMMAND ${CMAKE_COMMAND} -E copy
-    ${HYPERONC_TARGET_DIR}/${RUST_BUILD_SUFFIX}/${HYPERONC_SHARED_LIB_FILE}
-        ${HYPERONC_SHARED_LIB_PATH}
-    COMMAND ${CMAKE_COMMAND} -E copy
     ${HYPERONC_TARGET_DIR}/${RUST_BUILD_SUFFIX}/${HYPERONC_STATIC_LIB_FILE}
         ${HYPERONC_STATIC_LIB_PATH}
 
@@ -47,15 +50,6 @@ add_custom_target(build-hyperonc ALL
 # cmake checks INTERFACE_INCLUDE_DIRECTORIES contains existing path
 execute_process(COMMAND mkdir -p ${HYPERONC_TARGET_DIR})
 
-add_library(hyperonc-shared SHARED IMPORTED GLOBAL)
-set_target_properties(hyperonc-shared PROPERTIES
-    IMPORTED_LOCATION "${HYPERONC_SHARED_LIB_PATH}"
-    INTERFACE_INCLUDE_DIRECTORIES "${HYPERONC_TARGET_DIR}"
-    # required to import hyperonc-shared by name not by relative path
-    IMPORTED_NO_SONAME TRUE
-    )
-add_dependencies(hyperonc-shared build-hyperonc)
-
 add_library(hyperonc-static STATIC IMPORTED GLOBAL)
 set_target_properties(hyperonc-static PROPERTIES
     IMPORTED_LOCATION "${HYPERONC_STATIC_LIB_PATH}"
@@ -68,20 +62,51 @@ add_subdirectory(tests)
 set(BINARY_INSTALL_PATH "lib/hyperonc")
 set(INCLUDE_INSTALL_PATH "include/hyperonc")
 set(CONFIG_INSTALL_PATH "lib/cmake/hyperonc")
-set(SHARED_LIBRARY_INSTALL_PATH "${BINARY_INSTALL_PATH}/${HYPERONC_SHARED_LIB_FILE}")
 set(STATIC_LIBRARY_INSTALL_PATH "${BINARY_INSTALL_PATH}/${HYPERONC_STATIC_LIB_FILE}")
 
 include(CMakePackageConfigHelpers)
-configure_package_config_file("hyperonc-config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/hyperonc-config.cmake"
-    INSTALL_DESTINATION "${CONFIG_INSTALL_PATH}"
-    PATH_VARS INCLUDE_INSTALL_PATH SHARED_LIBRARY_INSTALL_PATH
-    STATIC_LIBRARY_INSTALL_PATH
-)
+
+if(BUILD_SHARED_LIBS)
+    set(HYPERONC_SHARED_LIB_FILE ${CMAKE_SHARED_LIBRARY_PREFIX}hyperonc${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(HYPERONC_SHARED_LIB_PATH ${HYPERONC_TARGET_DIR}/${HYPERONC_SHARED_LIB_FILE})
+    set(SHARED_LIBRARY_INSTALL_PATH "${BINARY_INSTALL_PATH}/${HYPERONC_SHARED_LIB_FILE}")
+
+    add_custom_target(copy-hyperonc-shared ALL
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${HYPERONC_TARGET_DIR}/${RUST_BUILD_SUFFIX}/${HYPERONC_SHARED_LIB_FILE}
+            ${HYPERONC_SHARED_LIB_PATH}
+
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    add_dependencies(copy-hyperonc-shared build-hyperonc)
+
+    add_library(hyperonc-shared SHARED IMPORTED GLOBAL)
+    set_target_properties(hyperonc-shared PROPERTIES
+        IMPORTED_LOCATION "${HYPERONC_SHARED_LIB_PATH}"
+        INTERFACE_INCLUDE_DIRECTORIES "${HYPERONC_TARGET_DIR}"
+        # required to import hyperonc-shared by name not by relative path
+        IMPORTED_NO_SONAME TRUE
+        )
+    add_dependencies(hyperonc-shared copy-hyperonc-shared build-hyperonc)
+
+    install(FILES "${HYPERONC_SHARED_LIB_PATH}"
+        DESTINATION "${BINARY_INSTALL_PATH}")
+
+    configure_package_config_file("hyperonc-config-shared.cmake.in"
+        "${CMAKE_CURRENT_BINARY_DIR}/hyperonc-config.cmake"
+        INSTALL_DESTINATION "${CONFIG_INSTALL_PATH}"
+        PATH_VARS INCLUDE_INSTALL_PATH SHARED_LIBRARY_INSTALL_PATH
+        STATIC_LIBRARY_INSTALL_PATH
+    )
+else(BUILD_SHARED_LIBS)
+    configure_package_config_file("hyperonc-config.cmake.in"
+        "${CMAKE_CURRENT_BINARY_DIR}/hyperonc-config.cmake"
+        INSTALL_DESTINATION "${CONFIG_INSTALL_PATH}"
+        PATH_VARS INCLUDE_INSTALL_PATH STATIC_LIBRARY_INSTALL_PATH
+    )
+endif(BUILD_SHARED_LIBS)
 
 install(FILES
     "${HYPERONC_STATIC_LIB_PATH}"
-    "${HYPERONC_SHARED_LIB_PATH}"
     DESTINATION "${BINARY_INSTALL_PATH}")
 install(DIRECTORY "${HYPERONC_INCLUDE_DIR}"
     DESTINATION "${INCLUDE_INSTALL_PATH}")

c/hyperonc-config-shared.cmake.in

@@ -0,0 +1,8 @@
+@PACKAGE_INIT@
+
+set_and_check(hyperonc_INCLUDE_DIRS "@PACKAGE_INCLUDE_INSTALL_PATH@")
+set_and_check(hyperonc_STATIC_LIBRARY "@PACKAGE_STATIC_LIBRARY_INSTALL_PATH@")
+set_and_check(hyperonc_SHARED_LIBRARY "@PACKAGE_SHARED_LIBRARY_INSTALL_PATH@")
+
+check_required_components(hyperonc)
+

c/hyperonc-config.cmake.in

@@ -2,7 +2,6 @@
 
 set_and_check(hyperonc_INCLUDE_DIRS "@PACKAGE_INCLUDE_INSTALL_PATH@")
 set_and_check(hyperonc_STATIC_LIBRARY "@PACKAGE_STATIC_LIBRARY_INSTALL_PATH@")
-set_and_check(hyperonc_SHARED_LIBRARY "@PACKAGE_SHARED_LIBRARY_INSTALL_PATH@")
 
 check_required_components(hyperonc)
 

c/src/atom.rs

@@ -361,7 +361,7 @@ pub extern "C" fn atom_gnd(gnd: *mut gnd_t) -> *mut atom_t {
 }
 
 /// Returns a new grounded `atom_t` referencing the space
-/// 
+///
 /// This function does not consume the space and the space still must be freed with `space_free`
 #[no_mangle]
 pub extern "C" fn atom_gnd_for_space(space: *mut space_t) -> *mut atom_t {
@@ -414,7 +414,7 @@ pub unsafe extern "C" fn atom_get_object(atom: *const atom_t) -> *mut gnd_t {
 }
 
 /// Returns a space_t from a grounded atom referencing the space
-/// 
+///
 /// The returned space is borrowed from the atom, and should not be freed nor accessed after the atom
 /// has been freed.
 #[no_mangle]

c/tests/CMakeLists.txt

@@ -5,17 +5,17 @@ set(TEST_SOURCES
     util.c
 )
 add_executable(check_atom check_atom.c ${TEST_SOURCES})
-target_link_libraries(check_atom hyperonc-shared CONAN_PKG::libcheck)
+target_link_libraries(check_atom hyperonc-static CONAN_PKG::libcheck)
 add_test(NAME check_atom COMMAND check_atom)
 
 add_executable(check_space check_space.c ${TEST_SOURCES})
-target_link_libraries(check_space hyperonc-shared CONAN_PKG::libcheck)
+target_link_libraries(check_space hyperonc-static CONAN_PKG::libcheck)
 add_test(NAME check_space COMMAND check_space)
 
 add_executable(check_sexpr_parser check_sexpr_parser.c ${TEST_SOURCES})
-target_link_libraries(check_sexpr_parser hyperonc-shared CONAN_PKG::libcheck)
+target_link_libraries(check_sexpr_parser hyperonc-static CONAN_PKG::libcheck)
 add_test(NAME check_sexpr_parser COMMAND check_sexpr_parser)
 
 add_executable(check_types check_types.c ${TEST_SOURCES})
-target_link_libraries(check_types hyperonc-shared CONAN_PKG::libcheck)
+target_link_libraries(check_types hyperonc-static CONAN_PKG::libcheck)
 add_test(NAME check_types COMMAND check_types)

python/.gitignore

@@ -4,3 +4,4 @@ __pycache__
 /build
 /*.so
 /*.dylib
+/wheelhouse

python/CMakeLists.txt

@@ -1,17 +1,30 @@
+# version supported by Ubuntu 18.04
+cmake_minimum_required(VERSION 3.10.2)
+project(hyperonpy)
+
+enable_testing()
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure)
+
+# Fix behavior of CMAKE_CXX_STANDARD when targeting macOS.
+if (POLICY CMP0025)
+    cmake_policy(SET CMP0025 NEW)
+endif ()
+set(CMAKE_CXX_STANDARD 11)
+
 if (${CMAKE_VERSION} GREATER_EQUAL "3.12")
     # The default value ("FIRST") prefers the installation with the highest
     # version. "ONLY" sticks to a virtualenv even when its version is smaller
     # which is usually expected by an user.
     if (NOT DEFINED Python3_FIND_VIRTUALENV)
         set(Python3_FIND_VIRTUALENV "ONLY")
     endif()
-    find_package(Python3 3.6 REQUIRED COMPONENTS Interpreter Development)
+    find_package(Python3 3.7 REQUIRED COMPONENTS Interpreter Development.Module)
 else() # support Ubuntu 18.04
     # pybind11 config looks for Python path again when deprecated PythonInterp
     # and PythonLibs packages are used. It should find the same version though
     # because PYTHON_EXECUTABLE is already set.
-    find_package(PythonInterp 3.6 REQUIRED)
-    find_package(PythonLibs 3.6 REQUIRED)
+    find_package(PythonInterp 3.7 REQUIRED)
+    find_package(PythonLibs 3.7 REQUIRED)
     # Python installation path should be modified via Python environment.
     # Following way of getting paths seems to be working (for Ubuntu as well).
     execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import sysconfig; print(sysconfig.get_path('platlib'), end='')" OUTPUT_VARIABLE Python3_SITEARCH)
@@ -22,18 +35,18 @@ message(STATUS "Python native modules installation path (Python3_SITEARCH): ${Py
 message(STATUS "Python modules installation path (Python3_SITELIB): ${Python3_SITELIB}")
 
 execute_process(
-    COMMAND conan install --build missing ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND conan install --build -- ${CMAKE_CURRENT_SOURCE_DIR}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 include(${CMAKE_CURRENT_BINARY_DIR}/conan_paths.cmake)
 
 find_package(pybind11 REQUIRED)
 find_package(optional-lite REQUIRED)
 include_directories(${nonstd_INCLUDE_DIRS})
+find_package(hyperonc REQUIRED HINTS ${HYPERONC_INSTALL_PREFIX})
+include_directories(${hyperonc_INCLUDE_DIRS})
 
-pybind11_add_module(hyperonpy SHARED hyperonpy.cpp)
-target_link_libraries(hyperonpy PRIVATE hyperonc-static)
-set_target_properties(hyperonpy PROPERTIES
-    LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+pybind11_add_module(hyperonpy MODULE ./hyperonpy.cpp)
+target_link_libraries(hyperonpy PRIVATE "${hyperonc_STATIC_LIBRARY}")
 
 set(PYTHONPATH "${CMAKE_CURRENT_SOURCE_DIR}")
 add_subdirectory(tests)

python/install-hyperonc.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+HYPERONC_URL=https://github.com/trueagi-io/hyperon-experimental.git
+HYPERONC_REV=main
+while getopts 'u:r:' opt; do
+    case "$opt" in
+        u)
+            HYPERONC_URL="$OPTARG"
+            ;;
+        r)
+            HYPERONC_REV="$OPTARG"
+            ;;
+        ?|h)
+            echo "Usage: $(basename $0) [-u hyperonc_repo_url] [-r hyperonc_revision]"
+            exit 1
+            ;;
+    esac
+done
+
+echo "hyperonc repository URL $HYPERONC_URL"
+echo "hyperonc revision $HYPERONC_REV"
+
+# This is to build subunit from Conan on CentOS based manylinux images.
+if test "$AUDITWHEEL_POLICY" = "manylinux2014"; then
+    yum install -y perl-devel
+fi
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh
+sh /tmp/rustup.sh -y && rm /tmp/rustup.sh
+export PATH="${PATH}:${HOME}/.cargo/bin"
+cargo install cbindgen
+
+python3 -m pip install conan==1.60.1 pip==23.1.2
+PATH="${PATH}:${HOME}/.local/bin"
+conan profile new --detect default
+
+mkdir -p ${HOME}/hyperonc
+cd ${HOME}/hyperonc
+git init
+git remote add origin $HYPERONC_URL
+git fetch origin $HYPERONC_REV
+git reset --hard FETCH_HEAD
+
+mkdir -p ${HOME}/hyperonc/c/build
+cd ${HOME}/hyperonc/c/build
+cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release ..
+make
+make check
+make install

python/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools==65.6.3"]
+requires = ["setuptools==65.6.3", "conan==1.60.1", "cmake==3.26.4"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -32,3 +32,8 @@ dev = [
 [tool.setuptools]
 packages = [ "hyperon" ]
 package-dir = { "hyperon" = "hyperon" }
+
+[tool.cibuildwheel]
+before-all = "./install-hyperonc.sh"
+skip = "*-musllinux_i686"
+test-command = "pytest ./tests"