Transition build system of cuda_cccl and cuda_parallel to scikit-buil…

…d-core (#3597)

* Rewrite of cuda_cccl build system

Transition from using setuptools to using scikit-build-core, so
that CMakeLists.txt is used to ensure that include is the same
as in the cuda toolkit.

Added test folder to test that the package works correctly.

* Make project.version metadata dynamic

Use suggestion by Bradley to use scikit-build-core regex facility
to extract version value from "_version.py"

* Delete existing cuda_cccl

* Rename new_cuda_cccl to cuda_cccl to become ready to open a PR

* Fix build break

* Expand tests for cuda_cccl

1. Use pytest fixture to reuse computed inc_paths
2. Split long test into smaller individual tests with descriptive names
3. Add tests to check that directories contain expected marker files (versions)
4. Avoid using c4l in favor of cccl

* Change cuda_parallel over to use scikit-build-core

* Delete unused _setup.py

python/cuda_cccl/.gitignore

@@ -1,2 +1,4 @@
-cuda/cccl/include
+build
+dist
 *egg-info
+*~

python/cuda_cccl/CMakeLists.txt

@@ -0,0 +1,35 @@
+cmake_minimum_required(VERSION 3.21...3.31 FATAL_ERROR)
+
+# include_guard(GLOBAL)
+
+project(
+    CCCL_HEADERS
+    VERSION ${SKBUILD_PROJECT_VERSION}
+    LANGUAGES C CXX
+    DESCRIPTION "Headers of NVIDIA CUDA Core Compute Libraries"
+)
+
+add_subdirectory(../.. _parent_cccl)
+
+find_package(CUB REQUIRED)
+find_package(Thrust REQUIRED)
+find_package(libcudacxx REQUIRED)
+
+set(_dest_incl_dir cuda/cccl/include)
+
+# No end slash: create ${_dest_inc_dir}/cub
+install(
+    DIRECTORY ${CUB_SOURCE_DIR}/cub
+    DESTINATION ${_dest_incl_dir}
+)
+# No end slash: create ${_dest_inc_dir}/thrust
+install(
+    DIRECTORY ${Thrust_SOURCE_DIR}/thrust
+    DESTINATION ${_dest_incl_dir}
+)
+# Slash at the end: copy content of
+#                   include/ into ${_dest_inc_dir}/
+install(
+    DIRECTORY ${libcudacxx_SOURCE_DIR}/include/
+    DESTINATION ${_dest_incl_dir}
+)

python/cuda_cccl/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/cuda_cccl/cuda/cccl/include/__init__.py

@@ -0,0 +1 @@
+# Intentionally empty

python/cuda_cccl/cuda/cccl/include_paths.py

@@ -57,7 +57,7 @@ def get_include_paths() -> IncludePaths:
 
     return IncludePaths(
         cuda=cuda_incl,
-        libcudacxx=cccl_incl / "libcudacxx",
+        libcudacxx=cccl_incl,
         cub=cccl_incl,
         thrust=cccl_incl,
     )

python/cuda_cccl/pyproject.toml

@@ -3,27 +3,49 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 [build-system]
-requires = ["setuptools>=61.0.0"]
-build-backend = "setuptools.build_meta"
+requires = ["scikit-build-core>=0.10"]
+build-backend = "scikit_build_core.build"
 
 [project]
 name = "cuda-cccl"
+dynamic = ["version"]
 description = "Experimental Package with CCCL headers to support JIT compilation"
 authors = [{ name = "NVIDIA Corporation" }]
 classifiers = [
   "Programming Language :: Python :: 3 :: Only",
   "Environment :: GPU :: NVIDIA CUDA",
   "License :: OSI Approved :: Apache Software License",
 ]
+license-files = ["LICENSE"]
 requires-python = ">=3.9"
-dynamic = ["version", "readme"]
+readme = { file = "README.md", content-type = "text/markdown" }
 
 [project.urls]
 Homepage = "https://github.com/NVIDIA/cccl"
 
-[tool.setuptools.dynamic]
-version = { attr = "cuda.cccl._version.__version__" }
-readme = { file = ["README.md"], content-type = "text/markdown" }
+[tool.scikit-build]
+minimum-version = "build-system.requires"
+build-dir = "build/{wheel_tag}"
 
-[tool.setuptools.package-data]
-cuda = ["cccl/include/**/*"]
+[tool.scikit-build.cmake]
+version = ">=3.21"
+args = []
+build-type = "Release"
+source-dir = "."
+
+[tool.scikit-build.ninja]
+version = ">=1.11"
+make-fallback = true
+
+[tool.scikit-build.wheel]
+py-api = "py3"
+platlib = ""
+
+[tool.scikit-build.wheel.packages]
+"cuda" = "cuda"
+"cuda/cccl" = "cuda/cccl"
+
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "cuda/cccl/_version.py"
+# use default regex

python/cuda_cccl/test/test_cuda_cccl.py

@@ -0,0 +1,63 @@
+import pytest
+from cuda import cccl
+
+
+def test_version():
+    v = cccl.__version__
+    assert isinstance(v, str)
+
+
+@pytest.fixture
+def inc_paths():
+    return cccl.get_include_paths()
+
+
+def test_headers_has_cuda(inc_paths):
+    assert hasattr(inc_paths, "cuda")
+
+
+def test_headers_has_cub(inc_paths):
+    assert hasattr(inc_paths, "cub")
+
+
+def test_headers_has_cudacxx(inc_paths):
+    assert hasattr(inc_paths, "libcudacxx")
+
+
+def test_headers_has_thrust(inc_paths):
+    assert hasattr(inc_paths, "thrust")
+
+
+def test_headers_as_tuple(inc_paths):
+    tpl = inc_paths.as_tuple()
+    assert len(tpl) == 4
+
+    thrust_, cub_, cudacxx_, cuda_ = tpl
+    assert cuda_ == inc_paths.cuda
+    assert cub_ == inc_paths.cub
+    assert cudacxx_ == inc_paths.libcudacxx
+    assert thrust_ == inc_paths.thrust
+
+
+def test_cub_version(inc_paths):
+    cub_dir = inc_paths.cub / "cub"
+    cub_version = cub_dir / "version.cuh"
+    assert cub_version.exists()
+
+
+def test_thrust_version(inc_paths):
+    thrust_dir = inc_paths.thrust / "thrust"
+    thrust_version = thrust_dir / "version.h"
+    assert thrust_version.exists()
+
+
+def test_cudacxx_version(inc_paths):
+    cudacxx_dir = inc_paths.libcudacxx / "cuda"
+    cudacxx_version = cudacxx_dir / "version"
+    assert cudacxx_version.exists()
+
+
+def test_nv_target(inc_paths):
+    nv_dir = inc_paths.libcudacxx / "nv"
+    nv_target = nv_dir / "target"
+    assert nv_target.exists()

python/cuda_parallel/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.21)
+
+project(
+    cuda_parallel
+#    VERSION ${SKBUILD_PROJECT_VERSION}
+    DESCRIPTION "Python package cuda_parallel"
+    LANGUAGES CUDA CXX
+)
+
+set(_cccl_root ../..)
+
+include(${_cccl_root}/cmake/AppendOptionIfAvailable.cmake)
+include(${_cccl_root}/cmake/CCCLConfigureTarget.cmake)
+include(${_cccl_root}/cmake/CCCLBuildCompilerTargets.cmake)
+cccl_build_compiler_targets()
+
+set(CCCL_ENABLE_C ON)
+set(CCCL_C_PARALLEL_LIBRARY_OUTPUT_DIRECTORY ${SKBUILD_PROJECT_NAME})
+add_subdirectory(${_cccl_root} _parent_cccl)
+
+install(
+    TARGETS cccl.c.parallel
+    DESTINATION cuda/parallel/experimental/cccl
+)

python/cuda_parallel/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/cuda_parallel/pyproject.toml

@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 [build-system]
-requires = ["setuptools>=61.0.0"]
-build-backend = "setuptools.build_meta"
+requires = ["scikit-build-core>=0.10"]
+build-backend = "scikit_build_core.build"
 
 [project]
 name = "cuda-parallel"
@@ -17,17 +17,41 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = ["cuda-cccl", "numba>=0.60.0", "cuda-python==12.*"]
-dynamic = ["version", "readme"]
+dynamic = ["version"]
+readme = { file = "README.md", content-type = "text/markdown" }
 
 [project.optional-dependencies]
 test = ["pytest", "pytest-xdist", "cupy-cuda12x", "typing_extensions"]
 
 [project.urls]
 Homepage = "https://developer.nvidia.com/"
 
-[tool.setuptools.dynamic]
-version = { attr = "cuda.parallel._version.__version__" }
-readme = { file = ["README.md"], content-type = "text/markdown" }
+[tool.scikit-build]
+minimum-version = "build-system.requires"
+build-dir = "build/{wheel_tag}"
+
+[tool.scikit-build.cmake]
+version = ">=3.21"
+args = []
+build-type = "Release"
+source-dir = "."
+
+[tool.scikit-build.ninja]
+version = ">=1.11"
+make-fallback = true
+
+[tool.scikit-build.wheel]
+py-api = "py3"
+
+[tool.scikit-build.wheel.packages]
+"cuda" = "cuda"
+"cuda/parallel" = "cuda/parallel"
+"cuda/parallel/experimental" = "cuda/parallel/experimental"
+
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "cuda/parallel/_version.py"
+# use default regex
 
 [tool.mypy]
 python_version = "3.10"