MAINTENANCE: Massage packaging for Debian systems

Add packaging dependencies for successful run of 'mk-build-deps', 'dkpg-buildpackage' on Debian based systems (Debian Buster, Debian Bullseye, Ubuntu Bionic, Ubuntu Focal, Ubuntu Hirsute).

Modify the tests - both 'runtests' and 'pytests' - to run to completion during CI testing on Debian based systems.

CMakeLists.txt

@@ -24,10 +24,12 @@
 # #####################################################################
 # ~~~
 
-cmake_minimum_required(VERSION 3.21)
+cmake_minimum_required(VERSION 3.22)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
 set(CMAKE_EXPORT_LINK_COMMANDS TRUE)
+set(CMAKE_DISABLE_SOURCE_CHANGES TRUE)
+set(CMAKE_DISABLE_IN_SOURCE_BUILD TRUE)
 
 project(
   Machinekit-HAL
@@ -42,6 +44,26 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   enable_testing()
 endif()
 
+if(CMAKE_TOOLCHAIN_FILE)
+  cmake_path(IS_RELATIVE CMAKE_TOOLCHAIN_FILE _toolchain_file_is_relative)
+  if(_toolchain_file_is_relative)
+    file(REAL_PATH "${CMAKE_TOOLCHAIN_FILE}" _toolchain_path BASE_DIRECTORY
+         "${CMAKE_BINARY_DIR}" EXPAND_TILDE)
+    if(NOT EXISTS "${_toolchain_path}")
+      message(
+        FATAL_ERROR "Specified Toolchain file ${_toolchain_path} doesn't exist!"
+      )
+    endif()
+    set(CMAKE_TOOLCHAIN_FILE "${_toolchain_path}")
+    message(
+      WARNING
+        "CMAKE_TOOLCHAIN_FILE set to ${CMAKE_TOOLCHAIN_FILE} to resolve the relative path issue. Please, avoid using relative paths!"
+    )
+    unset(_toolchain_path)
+  endif()
+  unset(_toolchain_file_is_relative)
+endif()
+
 string(REGEX REPLACE "[-~ .]" "_" MACHINEKIT_HAL_PACKAGE_PREFIX_NORMALIZED
                      ${CMAKE_PROJECT_NAME})
 string(TOLOWER ${MACHINEKIT_HAL_PACKAGE_PREFIX_NORMALIZED}
@@ -95,14 +117,15 @@ message(STATUS "Value of CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}")
 
 set(MACHINEKIT_HAL_GIT_BUILD_SHA "aaabbbccc") # TODO: Set based on actual Git
                                               # SHA
+
 define_property(
-  GLOBAL
-  PROPERTY MACHINEKIT_HAL_MODULE_PATH
-  BRIEF_DOCS "Machinekit-HAL specific CMake module PATH"
+  TARGET
+  PROPERTY "PUBLIC_INCLUDE_DIRECTORY"
+  BRIEF_DOCS "Header directory for a target"
   FULL_DOCS
     [[
-  Special global property specific to Machinekit-HAL's build tree holding the set of PATHs
-  to defined CMake Modules in the source-tree.
+    Special traget specific property for storing a path to directory where PUBLIC HEADERS
+    live, typically the "include/package_name" one
   ]])
 
 add_subdirectory(src)

debian/bootstrap

@@ -233,6 +233,23 @@ check_necessity_of_pasm() {
     return 0
 }
 
+check_necessity_of_importlib_resourses() {
+    local python="$(apt-cache show python3.6)"
+    local retval="$?" 
+    if ((retval != 0 ))
+    then
+        log_error "Command 'apt-cache show python3-importlib-resources' returned error code $retval!"
+        return -1
+    fi
+
+    if [[ ! -z "$python" ]]
+    then
+        BUILDTIME_DEPENDENCIES+=", python3-importlib-resources:native"
+    fi
+
+    return 0
+}
+
 test_dir(){
     local version_file="$1/VERSION"
     local readme_file="$1/README.md"
@@ -353,6 +370,12 @@ _main(){
     then
         failure
     fi
+    check_necessity_of_importlib_resourses
+    retval=$?
+    if (( retval != 0 ))
+    then
+        failure
+    fi
     generate_control_file
     retval=$?
     if (( retval != 0 ))

debian/buildsystem/Dockerfile

@@ -125,11 +125,11 @@ RUN curl -1vLf
         --output /tmp/cmake.sh &&                                                                                   \
     bash /tmp/cmake.sh --skip-license --prefix=/usr/local
 
-# Python 3.9 has --upgrade-deps, but lower versions needs manual action
+# Python 3.9 has --upgrade-deps, but lower versions need manual action
 RUN python3 -m venv /home/${USER}/pybuild        \
             --system-site-packages            && \
         /home/${USER}/pybuild/bin/pip install    \
-            --upgrade pip build setuptools
+            --upgrade pip setuptools
 
 RUN ln -s /home/${USER}/pybuild/bin/activate \
         /opt/environment/pyactivate

debian/control.in

@@ -48,9 +48,11 @@ Build-Depends:
     libprotoc-dev:native (>= 2.4.1),
     python3-simplejson:native,
     python3-sh:native,
+    python3-installer:native,
     python3-pytest:native,
     libcgroup-dev,
     yapps2:native,
+    python3-build:native,
     python3-yapps:native,
     python3-pyftpdlib:native,
     python3-pip:native,

debian/debianMultiarchToolchain.cmake

@@ -70,6 +70,13 @@ if(DEFINED ENV{DEB_BUILD_GNU_TYPE}
     set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
     set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
 
+    # Is the ARMHF platform the only one with a special name?
+    if("$ENV{DEB_HOST_GNU_CPU}" STREQUAL "arm")
+      set(PYTHON_PLAT_NAME "$ENV{DEB_HOST_ARCH_OS}-armv7l")
+    else()
+      set(PYTHON_PLAT_NAME "$ENV{DEB_HOST_ARCH_OS}-$ENV{DEB_HOST_GNU_CPU}")
+    endif()
+
     # Prepend the architecture specific include directories to the compiler
     # search path
     include_directories(BEFORE "/usr/$ENV{DEB_HOST_GNU_TYPE}/include"

debian/rules

@@ -29,7 +29,7 @@ DEBIAN_BULLSEYE_VERSION :=11
 
 #	Turn OFF the Link-Time Optimization (LTO) during compilation as it clashes with the
 #	current version of linker script turning symbols of HAL modules local (defined
-#	in 'MachinekitHALSymbolVisibility.cmake' CMake file)
+#	in 'MachinekitHALSymbolVisibilityFunction.cmake' CMake file)
 DEB_CFLAGS_MAINT_STRIP :=-flto=auto -ffat-lto-objects
 DEB_CXXFLAGS_MAINT_STRIP :=-flto=auto -ffat-lto-objects
 DEB_LDFLAGS_MAINT_STRIP :=-flto=auto
@@ -43,6 +43,19 @@ DEB_CXXFLAGS_MAINT_APPEND :=-O0
 #	Do not warn on potential reproducible build datetime mismatch
 DEB_CPPFLAGS_MAINT_STRIP :=-Wdate-time
 
+#	Python3 distro specific paths used for installation of Pythonic
+#	packages to the system
+#	TODO: Are these paths right?
+MACHINEKIT_HAL_PYTHON_STDLIB_FULL_INSTALL_DIRECTORY :=/usr/lib
+MACHINEKIT_HAL_PYTHON_PURELIB_FULL_INSTALL_DIRECTORY :=/usr/lib/python3/dist-packages
+MACHINEKIT_HAL_PYTHON_PLATSTDLIB_FULL_INSTALL_DIRECTORY :=/usr/lib/python3
+MACHINEKIT_HAL_PYTHON_PLATLIB_FULL_INSTALL_DIRECTORY :=/usr/lib/python3/dist-packages
+MACHINEKIT_HAL_PYTHON_INCLUDE_FULL_INSTALL_DIRECTORY :=/usr/include
+MACHINEKIT_HAL_PYTHON_PLATINCLUDE_FULL_INSTALL_DIRECTORY :=/usr/include
+MACHINEKIT_HAL_PYTHON_SCRIPTS_FULL_INSTALL_DIRECTORY :=/usr/bin
+MACHINEKIT_HAL_PYTHON_DATA_FULL_INSTALL_DIRECTORY :=/usr
+MACHINEKIT_HAL_PYTHON_INTERPRETER_INSTALL_EXECUTABLE :=/usr/bin/python3
+
 export DEB_CFLAGS_MAINT_STRIP
 export DEB_CXXFLAGS_MAINT_STRIP
 export DEB_LDFLAGS_MAINT_STRIP
@@ -52,7 +65,16 @@ export DEB_CPPFLAGS_MAINT_STRIP
 
 override_dh_auto_configure:
 	dh_auto_configure -- \
-		--toolchain $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))/debianMultiarchToolchain.cmake
+		--toolchain $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))/debianMultiarchToolchain.cmake \
+		-DMACHINEKIT_HAL_PYTHON_STDLIB_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_STDLIB_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_PURELIB_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_PURELIB_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_PLATSTDLIB_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_PLATSTDLIB_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_PLATLIB_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_PLATLIB_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_INCLUDE_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_INCLUDE_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_PLATINCLUDE_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_PLATINCLUDE_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_SCRIPTS_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_SCRIPTS_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_DATA_FULL_INSTALL_DIRECTORY=$(MACHINEKIT_HAL_PYTHON_DATA_FULL_INSTALL_DIRECTORY) \
+		-DMACHINEKIT_HAL_PYTHON_INTERPRETER_INSTALL_EXECUTABLE=$(MACHINEKIT_HAL_PYTHON_INTERPRETER_INSTALL_EXECUTABLE)
 
 override_dh_auto_test:
 #	Only run tests (CTest) when not cross-compiling

debian/runtests.py

@@ -43,6 +43,7 @@
 import shutil
 import pathlib
 import tempfile
+from contextlib import nullcontext
 from typing import Union, List
 
 import importlib.util
@@ -133,9 +134,11 @@ def run_runtests(self, test_path: pathlib.Path = None) -> None:
             test_path = test_path if isinstance(
                 test_path, pathlib.Path) else pathlib.Path(test_path)
         for config in self.configs:
-            config_test_path = self.build_directory / config if self.generator == "Ninja Multi-Config" else "." / \
+            config_test_path = self.build_directory / \
+                (config if self.generator == "Ninja Multi-Config" else ".") / \
                 "share" / "machinekit" / "hal" / "testsuite" / \
                 "runtests" if test_path is None else test_path
+
             test_directory = pathlib.Path(tempfile.mkdtemp(suffix=config))
             destination = test_directory / "runtests"
             print(
@@ -144,6 +147,7 @@ def run_runtests(self, test_path: pathlib.Path = None) -> None:
             shutil.copytree(config_test_path, destination, ignore=shutil.ignore_patterns(
                 'pipe-*', 'result', 'strerr'))
             bash_command_string = f". {self._envrc_path(config)}; run_runtests {destination}"
+
             sh.bash("-c", bash_command_string,
                     _out=sys.stdout.buffer, _err=sys.stderr.buffer)
 
@@ -152,13 +156,16 @@ def run_ctests(self) -> None:
             sh.ctest("-C", config, _cwd=self.build_directory,
                      _out=sys.stdout.buffer, _err=sys.stderr.buffer)
 
-    # def run_python_tests(self: object) -> None:
-    #    bash_command_string = ". {0}; ./nosetests/runpytest.sh".format(
-    #        self.rip_environment_path)
-    #    sh.bash("-c", bash_command_string,
-    #            _cwd=self.normalized_path,
-    #            _out=sys.stdout.buffer,
-    #            _err=sys.stderr.buffer)
+    def run_python_tests(self: object) -> None:
+        runpytest_path = self.source_directory / "tests" / "nosetests" / "runpytest.sh"
+        for config in self.configs:
+            print(
+                f"--->PyTests for config {config}.<---"
+                "\n====================================")
+        bash_command_string = f". {self._envrc_path(config)}; {runpytest_path}"
+        sh.bash("-c", bash_command_string,
+                _out=sys.stdout.buffer,
+                _err=sys.stderr.buffer)
 
     def configure(self,
                   cache: dict = dict()) -> None:
@@ -176,28 +183,28 @@ def configure(self,
                  _out=sys.stdout.buffer)
 
     def build(self,
-              target: str = None) -> None:
+              target: str = None,
+              sudo: bool = False) -> None:
         number_of_cores_string = sh.nproc(_tty_out=False).strip()
 
         build_additional = list()
         if target is not None:
             build_additional.append(["--target", target])
 
+        if sudo:
+            _context = sh.contrib.sudo(
+                password=self.sudo_password, _with=True, _out=sys.stdout.buffer)
+        else:
+            _context = nullcontext()
+
         for config in self.configs:
-            sh.cmake("--build", self.build_directory,
-                     "-j", number_of_cores_string,
-                     "--verbose",
-                     "--config", config,
-                     *build_additional,
-                     _out=sys.stdout.buffer)
-
-        # sh.Command(autogen_path)(_cwd=src_directory, _out=sys.stdout.buffer)
-        # configure_path = "{0}/configure".format(src_directory)
-        # sh.Command(configure_path)(_cwd=src_directory, _out=sys.stdout.buffer)
-        # sh.make("-j", number_of_cores_string,
-        #        _cwd=src_directory, _out=sys.stdout.buffer)
-        # sh.sudo("-S", "make", "setuid", _in=self.sudo_password,
-        #        _cwd=src_directory, _out=sys.stdout.buffer)
+            with _context:
+                sh.cmake("--build", self.build_directory,
+                         "-j", number_of_cores_string,
+                         "--verbose",
+                         "--config", config,
+                         *build_additional,
+                         _out=sys.stdout.buffer)
 
     def test_for_sudo(self: object) -> bool:
         try:
@@ -245,8 +252,10 @@ def main(args):
             build_script.configure()
         if not args.no_build:
             build_script.build()
-        # or args.no_python_tests):
-        if not (args.no_runtests or args.no_ctests):
+            build_script.build(target='binary_tree_venv')
+            build_script.build(target='setuid', sudo=True)
+
+        if not (args.no_runtests or args.no_ctests or args.no_python_tests):
             build_script.disable_zeroconf()
         if not args.no_runtests:
             try:
@@ -262,8 +271,13 @@ def main(args):
                 exception = True
                 print(e)
                 print("Run of Machinekit-HAL's CTests FAILED!")
-        # if not args.no_python_tests:
-        #    build_script.run_python_tests()
+        if not args.no_python_tests:
+            try:
+                build_script.run_python_tests()
+            except Exception as e:
+                exception = True
+                print(e)
+                print("Run of Machinekit-HAL's PyTests FAILED!")
     except ValueError as e:
         print(e)
         exception = True
@@ -349,6 +363,7 @@ def __call__(self, parser, namespace, values, option_string=None):
                         dest="sudo_password",
                         action="store",
                         metavar="PASSWORD",
+                        default="",
                         help="Password for usage with sudo command.")
 
     parser.add_argument("--no-configure",
@@ -367,9 +382,9 @@ def __call__(self, parser, namespace, values, option_string=None):
                         action="store_true",
                         help="Do not run CMake CTest tests")
 
-    # parser.add_argument("--no-python-tests",
-    #                    action="store_true",
-    #                    help="Do not run python tests")
+    parser.add_argument("--no-python-tests",
+                        action="store_true",
+                        help="Do not run python tests")
 
     args = parser.parse_args()
 

src/CMakeLists.txt

@@ -80,11 +80,12 @@ if(NOT MACHINEKIT_HAL_TEST_OUTPUT_DIRECTORY)
       "${MACHINEKIT_HAL_ARTIFACTS_MOUNTPOINT_DIRECTORY}/${MACHINEKIT_HAL_TEST_DIRECTORY}"
   )
 endif()
-# The export contains versions for all configuration, so there is no need for the configuration
-# specific ${MACHINEKIT_HAL_ARTIFACTS_MOUNTPOINT_DIRECTORY} (and it even cannot be used for generator
-# expression in it), but to have direct similarity to install TREE, use the './lib/<GNU-triplet>/cmake'
-# format even for BUILD tree (even though it may seem odd)
-# TODO: Investigate if this is really the best solution (as the relative path change does not matter much)
+# The export contains versions for all configuration, so there is no need for
+# the configuration specific ${MACHINEKIT_HAL_ARTIFACTS_MOUNTPOINT_DIRECTORY}
+# (and it even cannot be used for generator expression in it), but to have
+# direct similarity to install TREE, use the './lib/<GNU-triplet>/cmake' format
+# even for BUILD tree (even though it may seem odd) TODO: Investigate if this is
+# really the best solution (as the relative path change does not matter much)
 if(NOT MACHINEKIT_HAL_CMAKE_PACKAGE_OUTPUT_DIRECTORY)
   set(MACHINEKIT_HAL_CMAKE_PACKAGE_OUTPUT_DIRECTORY
       "${CMAKE_BINARY_DIR}/${MACHINEKIT_HAL_CMAKE_PACKAGE_DIRECTORY}")
@@ -185,6 +186,13 @@ if(BUILD_PYTHON_DISTRIBUTIONS)
     COMPONENTS Interpreter Development
     REQUIRED)
 
+  set(MACHINEKIT_HAL_PYTHON_INTERPRETER_INSTALL_EXECUTABLE
+      "${Python_EXECUTABLE}"
+      CACHE
+        FILEPATH
+        "Path to the Python3 interpreter which will be used as main execution interpreter for installed COMPONENTS."
+  )
+
   if(NOT MACHINEKIT_HAL_PYTHON_STDLIB_FULL_INSTALL_DIRECTORY)
     set(MACHINEKIT_HAL_PYTHON_STDLIB_FULL_INSTALL_DIRECTORY "${Python_STDLIB}")
   endif()
@@ -225,6 +233,11 @@ if(BUILD_PYTHON_DISTRIBUTIONS)
   include(PEP503PythonIndex)
 endif()
 
+list(APPEND CMAKE_MODULE_PATH
+     "${MACHINEKIT_HAL_CMAKE_PACKAGE_OUTPUT_DIRECTORY}")
+list(APPEND CMAKE_PREFIX_PATH
+     "${MACHINEKIT_HAL_ARTIFACTS_MOUNTPOINT_DIRECTORY}")
+
 add_subdirectory(libraries)
 add_subdirectory(executables)
 add_subdirectory(modules)

src/executables/CMakeLists.txt

@@ -24,9 +24,7 @@
 # #####################################################################
 # ~~~
 
-add_custom_target(
-  setuid
-  COMMENT "Setting the setuid bit on selected targets")
+add_custom_target(setuid COMMENT "Setting the setuid bit on selected targets")
 
 add_subdirectory(comp)
 add_subdirectory(decode_protobuf)