From c52fcd0cdc6159c35da9368c70b63443a5a5e4b9 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:18:28 -0600 Subject: [PATCH] Transition build system of cuda_cccl and cuda_parallel to scikit-build-core (#3597) * Rewrite of cuda_cccl build system Transition from using setuptools to using scikit-build-core, so that CMakeLists.txt is used to ensure that include is the same as in the cuda toolkit. Added test folder to test that the package works correctly. * Make project.version metadata dynamic Use suggestion by Bradley to use scikit-build-core regex facility to extract version value from "_version.py" * Delete existing cuda_cccl * Rename new_cuda_cccl to cuda_cccl to become ready to open a PR * Fix build break * Expand tests for cuda_cccl 1. Use pytest fixture to reuse computed inc_paths 2. Split long test into smaller individual tests with descriptive names 3. Add tests to check that directories contain expected marker files (versions) 4. Avoid using c4l in favor of cccl * Change cuda_parallel over to use scikit-build-core * Delete unused _setup.py --- python/cuda_cccl/.gitignore | 4 +- python/cuda_cccl/CMakeLists.txt | 35 +++++++++++ python/cuda_cccl/LICENSE | 1 + .../cuda_cccl/cuda/cccl/include/__init__.py | 1 + python/cuda_cccl/cuda/cccl/include_paths.py | 2 +- python/cuda_cccl/pyproject.toml | 38 ++++++++--- python/cuda_cccl/setup.py | 51 --------------- python/cuda_cccl/test/test_cuda_cccl.py | 63 +++++++++++++++++++ python/cuda_parallel/CMakeLists.txt | 24 +++++++ python/cuda_parallel/LICENSE | 1 + .../cuda/parallel/experimental/cccl/.gitkeep | 0 python/cuda_parallel/pyproject.toml | 36 +++++++++-- python/cuda_parallel/setup.py | 49 --------------- 13 files changed, 189 insertions(+), 116 deletions(-) create mode 100644 python/cuda_cccl/CMakeLists.txt create mode 120000 python/cuda_cccl/LICENSE create mode 100644 python/cuda_cccl/cuda/cccl/include/__init__.py delete mode 100644 python/cuda_cccl/setup.py create mode 100644 python/cuda_cccl/test/test_cuda_cccl.py create mode 100644 python/cuda_parallel/CMakeLists.txt create mode 120000 python/cuda_parallel/LICENSE create mode 100644 python/cuda_parallel/cuda/parallel/experimental/cccl/.gitkeep delete mode 100644 python/cuda_parallel/setup.py diff --git a/python/cuda_cccl/.gitignore b/python/cuda_cccl/.gitignore index 24ec757199f..0ae14ff5c8f 100644 --- a/python/cuda_cccl/.gitignore +++ b/python/cuda_cccl/.gitignore @@ -1,2 +1,4 @@ -cuda/cccl/include +build +dist *egg-info +*~ diff --git a/python/cuda_cccl/CMakeLists.txt b/python/cuda_cccl/CMakeLists.txt new file mode 100644 index 00000000000..92311564aa3 --- /dev/null +++ b/python/cuda_cccl/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required(VERSION 3.21...3.31 FATAL_ERROR) + +# include_guard(GLOBAL) + +project( + CCCL_HEADERS + VERSION ${SKBUILD_PROJECT_VERSION} + LANGUAGES C CXX + DESCRIPTION "Headers of NVIDIA CUDA Core Compute Libraries" +) + +add_subdirectory(../.. _parent_cccl) + +find_package(CUB REQUIRED) +find_package(Thrust REQUIRED) +find_package(libcudacxx REQUIRED) + +set(_dest_incl_dir cuda/cccl/include) + +# No end slash: create ${_dest_inc_dir}/cub +install( + DIRECTORY ${CUB_SOURCE_DIR}/cub + DESTINATION ${_dest_incl_dir} +) +# No end slash: create ${_dest_inc_dir}/thrust +install( + DIRECTORY ${Thrust_SOURCE_DIR}/thrust + DESTINATION ${_dest_incl_dir} +) +# Slash at the end: copy content of +# include/ into ${_dest_inc_dir}/ +install( + DIRECTORY ${libcudacxx_SOURCE_DIR}/include/ + DESTINATION ${_dest_incl_dir} +) diff --git a/python/cuda_cccl/LICENSE b/python/cuda_cccl/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/cuda_cccl/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/cuda_cccl/cuda/cccl/include/__init__.py b/python/cuda_cccl/cuda/cccl/include/__init__.py new file mode 100644 index 00000000000..bb7b160deb3 --- /dev/null +++ b/python/cuda_cccl/cuda/cccl/include/__init__.py @@ -0,0 +1 @@ +# Intentionally empty diff --git a/python/cuda_cccl/cuda/cccl/include_paths.py b/python/cuda_cccl/cuda/cccl/include_paths.py index da8246b9195..ba626f3a5e7 100644 --- a/python/cuda_cccl/cuda/cccl/include_paths.py +++ b/python/cuda_cccl/cuda/cccl/include_paths.py @@ -57,7 +57,7 @@ def get_include_paths() -> IncludePaths: return IncludePaths( cuda=cuda_incl, - libcudacxx=cccl_incl / "libcudacxx", + libcudacxx=cccl_incl, cub=cccl_incl, thrust=cccl_incl, ) diff --git a/python/cuda_cccl/pyproject.toml b/python/cuda_cccl/pyproject.toml index ada06301a4c..f0e49b4e38c 100644 --- a/python/cuda_cccl/pyproject.toml +++ b/python/cuda_cccl/pyproject.toml @@ -3,11 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [build-system] -requires = ["setuptools>=61.0.0"] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core>=0.10"] +build-backend = "scikit_build_core.build" [project] name = "cuda-cccl" +dynamic = ["version"] description = "Experimental Package with CCCL headers to support JIT compilation" authors = [{ name = "NVIDIA Corporation" }] classifiers = [ @@ -15,15 +16,36 @@ classifiers = [ "Environment :: GPU :: NVIDIA CUDA", "License :: OSI Approved :: Apache Software License", ] +license-files = ["LICENSE"] requires-python = ">=3.9" -dynamic = ["version", "readme"] +readme = { file = "README.md", content-type = "text/markdown" } [project.urls] Homepage = "https://github.com/NVIDIA/cccl" -[tool.setuptools.dynamic] -version = { attr = "cuda.cccl._version.__version__" } -readme = { file = ["README.md"], content-type = "text/markdown" } +[tool.scikit-build] +minimum-version = "build-system.requires" +build-dir = "build/{wheel_tag}" -[tool.setuptools.package-data] -cuda = ["cccl/include/**/*"] +[tool.scikit-build.cmake] +version = ">=3.21" +args = [] +build-type = "Release" +source-dir = "." + +[tool.scikit-build.ninja] +version = ">=1.11" +make-fallback = true + +[tool.scikit-build.wheel] +py-api = "py3" +platlib = "" + +[tool.scikit-build.wheel.packages] +"cuda" = "cuda" +"cuda/cccl" = "cuda/cccl" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "cuda/cccl/_version.py" +# use default regex diff --git a/python/cuda_cccl/setup.py b/python/cuda_cccl/setup.py deleted file mode 100644 index f6e5e3fa033..00000000000 --- a/python/cuda_cccl/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. -# -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import shutil -from pathlib import Path - -from setuptools import setup -from setuptools.command.build_py import build_py - -PROJECT_PATH = Path(__file__).resolve().parent -CCCL_PATH = PROJECT_PATH.parents[1] - - -class CustomBuildPy(build_py): - """Copy CCCL headers BEFORE super().run() - - Note that the CCCL headers cannot be referenced directly: - setuptools (and pyproject.toml) does not support relative paths that - reference files outside the package directory (like ../../). - This is a restriction designed to avoid inadvertently packaging files - that are outside the source tree. - """ - - def run(self): - cccl_headers = [ - ("cub", "cub"), - ("libcudacxx", "include"), - ("thrust", "thrust"), - ] - - inc_path = PROJECT_PATH / "cuda" / "cccl" / "include" - inc_path.mkdir(parents=True, exist_ok=True) - - for proj_dir, header_dir in cccl_headers: - src_path = CCCL_PATH / proj_dir / header_dir - dst_path = inc_path / proj_dir - if dst_path.exists(): - shutil.rmtree(dst_path) - shutil.copytree(src_path, dst_path) - - init_py_path = inc_path / "__init__.py" - init_py_path.write_text("# Intentionally empty.\n") - - super().run() - - -setup( - license_files=["../../LICENSE"], - cmdclass={"build_py": CustomBuildPy}, -) diff --git a/python/cuda_cccl/test/test_cuda_cccl.py b/python/cuda_cccl/test/test_cuda_cccl.py new file mode 100644 index 00000000000..cf673b3c89c --- /dev/null +++ b/python/cuda_cccl/test/test_cuda_cccl.py @@ -0,0 +1,63 @@ +import pytest +from cuda import cccl + + +def test_version(): + v = cccl.__version__ + assert isinstance(v, str) + + +@pytest.fixture +def inc_paths(): + return cccl.get_include_paths() + + +def test_headers_has_cuda(inc_paths): + assert hasattr(inc_paths, "cuda") + + +def test_headers_has_cub(inc_paths): + assert hasattr(inc_paths, "cub") + + +def test_headers_has_cudacxx(inc_paths): + assert hasattr(inc_paths, "libcudacxx") + + +def test_headers_has_thrust(inc_paths): + assert hasattr(inc_paths, "thrust") + + +def test_headers_as_tuple(inc_paths): + tpl = inc_paths.as_tuple() + assert len(tpl) == 4 + + thrust_, cub_, cudacxx_, cuda_ = tpl + assert cuda_ == inc_paths.cuda + assert cub_ == inc_paths.cub + assert cudacxx_ == inc_paths.libcudacxx + assert thrust_ == inc_paths.thrust + + +def test_cub_version(inc_paths): + cub_dir = inc_paths.cub / "cub" + cub_version = cub_dir / "version.cuh" + assert cub_version.exists() + + +def test_thrust_version(inc_paths): + thrust_dir = inc_paths.thrust / "thrust" + thrust_version = thrust_dir / "version.h" + assert thrust_version.exists() + + +def test_cudacxx_version(inc_paths): + cudacxx_dir = inc_paths.libcudacxx / "cuda" + cudacxx_version = cudacxx_dir / "version" + assert cudacxx_version.exists() + + +def test_nv_target(inc_paths): + nv_dir = inc_paths.libcudacxx / "nv" + nv_target = nv_dir / "target" + assert nv_target.exists() diff --git a/python/cuda_parallel/CMakeLists.txt b/python/cuda_parallel/CMakeLists.txt new file mode 100644 index 00000000000..c42f4119e8e --- /dev/null +++ b/python/cuda_parallel/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.21) + +project( + cuda_parallel +# VERSION ${SKBUILD_PROJECT_VERSION} + DESCRIPTION "Python package cuda_parallel" + LANGUAGES CUDA CXX +) + +set(_cccl_root ../..) + +include(${_cccl_root}/cmake/AppendOptionIfAvailable.cmake) +include(${_cccl_root}/cmake/CCCLConfigureTarget.cmake) +include(${_cccl_root}/cmake/CCCLBuildCompilerTargets.cmake) +cccl_build_compiler_targets() + +set(CCCL_ENABLE_C ON) +set(CCCL_C_PARALLEL_LIBRARY_OUTPUT_DIRECTORY ${SKBUILD_PROJECT_NAME}) +add_subdirectory(${_cccl_root} _parent_cccl) + +install( + TARGETS cccl.c.parallel + DESTINATION cuda/parallel/experimental/cccl +) diff --git a/python/cuda_parallel/LICENSE b/python/cuda_parallel/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/cuda_parallel/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/cuda_parallel/cuda/parallel/experimental/cccl/.gitkeep b/python/cuda_parallel/cuda/parallel/experimental/cccl/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cuda_parallel/pyproject.toml b/python/cuda_parallel/pyproject.toml index e7d2b9f0081..c32f85e1ab8 100644 --- a/python/cuda_parallel/pyproject.toml +++ b/python/cuda_parallel/pyproject.toml @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [build-system] -requires = ["setuptools>=61.0.0"] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core>=0.10"] +build-backend = "scikit_build_core.build" [project] name = "cuda-parallel" @@ -17,7 +17,8 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = ["cuda-cccl", "numba>=0.60.0", "cuda-python==12.*"] -dynamic = ["version", "readme"] +dynamic = ["version"] +readme = { file = "README.md", content-type = "text/markdown" } [project.optional-dependencies] test = ["pytest", "pytest-xdist", "cupy-cuda12x", "typing_extensions"] @@ -25,9 +26,32 @@ test = ["pytest", "pytest-xdist", "cupy-cuda12x", "typing_extensions"] [project.urls] Homepage = "https://developer.nvidia.com/" -[tool.setuptools.dynamic] -version = { attr = "cuda.parallel._version.__version__" } -readme = { file = ["README.md"], content-type = "text/markdown" } +[tool.scikit-build] +minimum-version = "build-system.requires" +build-dir = "build/{wheel_tag}" + +[tool.scikit-build.cmake] +version = ">=3.21" +args = [] +build-type = "Release" +source-dir = "." + +[tool.scikit-build.ninja] +version = ">=1.11" +make-fallback = true + +[tool.scikit-build.wheel] +py-api = "py3" + +[tool.scikit-build.wheel.packages] +"cuda" = "cuda" +"cuda/parallel" = "cuda/parallel" +"cuda/parallel/experimental" = "cuda/parallel/experimental" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "cuda/parallel/_version.py" +# use default regex [tool.mypy] python_version = "3.10" diff --git a/python/cuda_parallel/setup.py b/python/cuda_parallel/setup.py deleted file mode 100644 index c5c9fcd3c32..00000000000 --- a/python/cuda_parallel/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. -# -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import subprocess -from pathlib import Path - -from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext - -CCCL_PYTHON_PATH = Path(__file__).resolve().parents[1] -CCCL_PATH = CCCL_PYTHON_PATH.parent - - -class CMakeExtension(Extension): - def __init__(self, name): - super().__init__(name, sources=[]) - - -class BuildCMakeExtension(build_ext): - def run(self): - for ext in self.extensions: - self.build_extension(ext) - - def build_extension(self, ext): - extdir = Path(self.get_ext_fullpath(ext.name)).resolve().parent - cmake_args = [ - "-DCCCL_ENABLE_C=YES", - f"-DCCCL_C_PARALLEL_LIBRARY_OUTPUT_DIRECTORY={extdir}", - "-DCMAKE_BUILD_TYPE=Release", - ] - - build_temp_path = Path(self.build_temp) - build_temp_path.mkdir(parents=True, exist_ok=True) - - subprocess.check_call(["cmake", CCCL_PATH] + cmake_args, cwd=build_temp_path) - subprocess.check_call( - ["cmake", "--build", ".", "--target", "cccl.c.parallel"], - cwd=build_temp_path, - ) - - -setup( - license_files=["../../LICENSE"], - cmdclass={ - "build_ext": BuildCMakeExtension, - }, - ext_modules=[CMakeExtension("cuda.parallel.experimental.cccl.c")], -)