From 27520a01de98da32764d33a2bf1f2a4e9e9388ad Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 30 Sep 2026 14:19:15 -0700 Subject: [PATCH 1/9] ci: build Cython test extensions inside cibuildwheel Reuse the wheel Cython cache in tests/cython/build_tests.py so the new test stage can hit CUDA_PYTHON_CYTHON_CACHE_DIR. (cherry picked from commit 392178690f26c7d7cb8117a5556a70f2c728c321) --- cuda_bindings/tests/cython/build_tests.py | 63 ++++++++++++++++++--- cuda_core/tests/cython/build_tests.py | 69 +++++++++++++++++++---- 2 files changed, 115 insertions(+), 17 deletions(-) diff --git a/cuda_bindings/tests/cython/build_tests.py b/cuda_bindings/tests/cython/build_tests.py index 5bde350e87b..1ea736f6749 100644 --- a/cuda_bindings/tests/cython/build_tests.py +++ b/cuda_bindings/tests/cython/build_tests.py @@ -8,19 +8,39 @@ so `cimport cuda.bindings.*` fails to locate the .pxd files. We resolve the namespace package's source root from `cuda.bindings.__file__` and pass it via `include_path=` so cythonize finds the .pxd tree on every platform. + +When CUDA_PYTHON_CYTHON_CACHE_DIR is set, cythonize uses the same cache +namespacing and include-path aliasing as cuda_bindings/build_hooks.py. """ from __future__ import annotations +import importlib.util import os import sys from pathlib import Path +import Cython from Cython.Build import cythonize from setuptools import setup import cuda.bindings +_COMPILER_DIRECTIVES = {"freethreading_compatible": True} + + +def _load_build_hooks(): + # PEP 517 backend, not an installed module. Load by path so we do not put + # cuda_bindings/ on sys.path (that would shadow the installed package). + build_hooks_path = Path(__file__).resolve().parents[2] / "build_hooks.py" + spec = importlib.util.spec_from_file_location("cuda_bindings_build_hooks", build_hooks_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +build_hooks = _load_build_hooks() + def _bindings_source_root() -> Path: # cuda.bindings.__file__ -> ...//cuda/bindings/__init__.py @@ -32,6 +52,41 @@ def _bindings_source_root() -> Path: return root +def _cythonize_tests(pyx_files): + cache_path = build_hooks._cython_cache_path( + "cuda-bindings-cython-tests", + compiler_directives=_COMPILER_DIRECTIVES, + language_level=3, + cplus=True, + ) + cythonize_kwargs = { + "language_level": 3, + "nthreads": 1, + "compiler_directives": _COMPILER_DIRECTIVES, + "cache": cache_path, + } + if cache_path is None: + return cythonize( + pyx_files, + include_path=[str(_bindings_source_root())], + **cythonize_kwargs, + ) + + # Distinct alias names so a concurrent package build's .cython-stdlib / + # .cython-bindings symlinks are not replaced. Relative aliases resolve + # next to build_hooks.py (package root). + stdlib_target = Path(Cython.__file__).parent / "Includes" + with build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib: + with build_hooks._stable_cython_alias( + _bindings_source_root(), Path(".cython-bindings-tests") + ) as rel_bindings: + return cythonize( + pyx_files, + include_path=[".", rel_bindings, rel_stdlib], + **cythonize_kwargs, + ) + + def main() -> None: script_dir = Path(__file__).resolve().parent # Avoid appending the absolute checkout path under build/temp: the @@ -41,13 +96,7 @@ def main() -> None: if not pyx_files: raise SystemExit(f"no test_*.pyx files under {script_dir}") - ext_modules = cythonize( - pyx_files, - language_level=3, - nthreads=1, - include_path=[str(_bindings_source_root())], - compiler_directives={"freethreading_compatible": True}, - ) + ext_modules = _cythonize_tests(pyx_files) # pytest imports each extension by bare module name (see test_cython.py), # so build in-place next to its .pyx regardless of the invoking cwd. diff --git a/cuda_core/tests/cython/build_tests.py b/cuda_core/tests/cython/build_tests.py index 6cebb0c6ff9..6518e9a679b 100644 --- a/cuda_core/tests/cython/build_tests.py +++ b/cuda_core/tests/cython/build_tests.py @@ -8,19 +8,39 @@ so `cimport cuda.bindings.*` fails to locate the .pxd files. We resolve the namespace package's source root from `cuda.bindings.__file__` and pass it via `include_path=` so cythonize finds the .pxd tree on every platform. + +When CUDA_PYTHON_CYTHON_CACHE_DIR is set, cythonize uses the same cache +namespacing and include-path aliasing as cuda_core/build_hooks.py. """ from __future__ import annotations +import importlib.util import os import sys from pathlib import Path +import Cython from Cython.Build import cythonize from setuptools import setup import cuda.bindings +_COMPILER_DIRECTIVES = {"freethreading_compatible": True} + + +def _load_build_hooks(): + # PEP 517 backend, not an installed module. Load by path so we do not put + # cuda_core/ on sys.path (that would shadow the installed package). + build_hooks_path = Path(__file__).resolve().parents[2] / "build_hooks.py" + spec = importlib.util.spec_from_file_location("cuda_core_build_hooks", build_hooks_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +build_hooks = _load_build_hooks() + def _bindings_source_root() -> Path: # cuda.bindings.__file__ -> ...//cuda/bindings/__init__.py @@ -32,26 +52,55 @@ def _bindings_source_root() -> Path: return root +def _cythonize_tests(pyx_files): + cache_path = build_hooks._cython_cache_path( + "cuda-core-cython-tests", + compiler_directives=_COMPILER_DIRECTIVES, + language_level=3, + cplus=True, + ) + cythonize_kwargs = { + "language_level": 3, + "compiler_directives": _COMPILER_DIRECTIVES, + "cache": cache_path, + } + if cache_path is None: + return cythonize( + pyx_files, + include_path=[str(_bindings_source_root())], + **cythonize_kwargs, + ) + + # Distinct alias names so a concurrent package build's .cython-stdlib / + # .cython-bindings symlinks are not replaced. Relative aliases resolve + # next to build_hooks.py (package root). + stdlib_target = Path(Cython.__file__).parent / "Includes" + with build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib: + with build_hooks._stable_cython_alias( + _bindings_source_root(), Path(".cython-bindings-tests") + ) as rel_bindings: + return cythonize( + pyx_files, + include_path=[".", rel_bindings, rel_stdlib], + **cythonize_kwargs, + ) + + def main() -> None: script_dir = Path(__file__).resolve().parent - pyx_files = sorted(str(p) for p in script_dir.glob("test_*.pyx")) + pyx_files = sorted(p.name for p in script_dir.glob("test_*.pyx")) if not pyx_files: raise SystemExit(f"no test_*.pyx files under {script_dir}") - ext_modules = cythonize( - pyx_files, - language_level=3, - include_path=[str(_bindings_source_root())], - compiler_directives={"freethreading_compatible": True}, - ) - # `build_ext --inplace` places the compiled .so relative to the current # working directory, but pixi runs this task from the project root. pytest # imports each extension by bare module name (see test_cython.py), which # only resolves when the .so sits in tests/cython (the dir pytest puts on - # sys.path). chdir here so the .so lands next to its .pyx regardless of the - # invoking cwd. + # sys.path). chdir before cythonize so alias relpaths and the .so location + # are stable regardless of the invoking cwd. os.chdir(script_dir) + ext_modules = _cythonize_tests(pyx_files) + sys.argv = [sys.argv[0], "build_ext", "--inplace"] setup(name="cuda_core_cython_tests", ext_modules=ext_modules) From cf2ca95b13ffad25cbf92d16d15afa6c42c9ab84 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 07:56:30 -0700 Subject: [PATCH 2/9] build: unify compiler flag sets across cuda-bindings and cuda-core Close the audit from #1882. The two packages' Linux and MSVC flag sets had drifted for legacy reasons; both now go through the same structure. Changes in cuda-bindings: - Drop -fpermissive and -fno-var-tracking-assignments (gcc-only; not needed by current Cython-generated C++; confirmed by a rebuild). - Change -O3 to -O2 (consistent with cuda-core; -O3 was not the source of the measured launch_{256,512}_args latency difference vs c++17). - Add /std:c++14 and /O2 on MSVC (was missing entirely). - Split '-std=c++14 -Wno-deprecated-declarations' onto separate lines with an explanatory comment on each. Changes in cuda-core: - Add /O2 on MSVC (modern setuptools no longer forces /Ox). - Add comments at both the MSVC and Linux flag sites explaining why c++17 is required (structured bindings / if constexpr in _cpp/). Tests: - Update the bindings TestResolveToolchain: replace test_gnu_keeps_gcc_only_flags with test_linux_opt_flag_set (asserts -O2, not -O3; no gnu-only flags); add test_msvc_opt_flag_set; update test_gnu_sets_env_and_flags assertions. - Add test_linux_opt_flag_set and test_msvc_opt_flag_set to the core TestResolveToolchain; update the comment in test_gnu_sets_env_and_flags. --- cuda_bindings/build_hooks.py | 17 ++++++++---- cuda_bindings/tests/test_build_hooks.py | 36 ++++++++++++++++++------- cuda_core/build_hooks.py | 4 ++- cuda_core/tests/test_build_hooks.py | 28 +++++++++++++++++-- 4 files changed, 67 insertions(+), 18 deletions(-) diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index a1f572d7a37..eab8f7c6299 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -437,21 +437,28 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False): extra_link_args = [] if name == "msvc": + # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args + # from gcc's c++17 variadic-template expansion. + extra_compile_args += ["/std:c++14", "/O2"] if debug: raise RuntimeError("Debuggable builds are not supported on Windows.") else: # Common Linux compile flags. - extra_compile_args += ["-std=c++14", "-Wno-deprecated-declarations"] + # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args + # from gcc's c++17 variadic-template expansion. + extra_compile_args += ["-std=c++14"] + # cudaMemcpy*Array* and cudaGetDriverEntryPoint are deprecated but still + # supported; suppress the resulting warnings so a future -Werror build + # is not broken by Cython-generated calls we cannot control. + extra_compile_args += ["-Wno-deprecated-declarations"] # Compiler-specific flags. - if name == "gnu": - extra_compile_args += ["-fpermissive", "-fno-var-tracking-assignments"] - elif name == "llvm": + if name == "llvm": extra_link_args += ["-fuse-ld=lld"] # Common Linux debug/opt flags. if debug: extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] else: - extra_compile_args += ["-g0", "-O3"] + extra_compile_args += ["-g0", "-O2"] extra_link_args += ["-Wl,--strip-all"] if compile_for_coverage: diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index 78c41ed1885..b7606f997ba 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -148,9 +148,8 @@ def test_gnu_sets_env_and_flags(self, monkeypatch): assert os.environ["CXX"] == "g++" assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" assert "LDSHARED" not in os.environ - # gcc-only flags are present (this is the point of P2: explicit gnu must use gcc, not generic cc) - assert "-fpermissive" in cargs - assert "-fno-var-tracking-assignments" in cargs + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs @pytest.mark.agent_authored(model="grok-4.6") def test_llvm_keeps_sccache_prefix(self, monkeypatch): @@ -167,14 +166,31 @@ def test_llvm_keeps_sccache_prefix(self, monkeypatch): # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - @pytest.mark.agent_authored(model="glm-5.2") - def test_gnu_keeps_gcc_only_flags(self, monkeypatch): + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_linux_opt_flag_set(self, monkeypatch): + """Linux opt build: -std=c++14, -O2, -Wno-deprecated-declarations; no -O3 or gnu-only flags.""" if sys.platform == "win32": - pytest.skip("gnu only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") - _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain() - assert "-fpermissive" in cargs - assert "-fno-var-tracking-assignments" in cargs + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain(debug=False) + assert "-std=c++14" in cargs + assert "-Wno-deprecated-declarations" in cargs + assert "-g0" in cargs + assert "-O2" in cargs + assert "-O3" not in cargs + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + assert "-Wl,--strip-all" in largs + + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_msvc_opt_flag_set(self, monkeypatch): + """MSVC opt build emits /std:c++14 and /O2.""" + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "/std:c++14" in cargs + assert "/O2" in cargs @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index afa956fa4d1..cf09a1bd288 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -484,11 +484,13 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False): extra_link_args = [] if name == "msvc": - extra_compile_args += ["/std:c++17"] + # c++17: required by structured bindings and if constexpr in cuda/core/_cpp/. + extra_compile_args += ["/std:c++17", "/O2"] if debug: raise RuntimeError("Debuggable builds are not supported on Windows.") else: # Common Linux compile flags. + # c++17: required by structured bindings and if constexpr in cuda/core/_cpp/. extra_compile_args += ["-std=c++17"] # Compiler-specific flags. if name == "llvm": diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 7e9499f89e3..f5a431c9810 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -976,8 +976,8 @@ def test_gnu_sets_env_and_flags(self, monkeypatch): assert os.environ["CXX"] == "g++" assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" assert "LDSHARED" not in os.environ - # gcc-only flags are present (this is the point of P2: explicit gnu must use gcc, not generic cc) - assert "-fpermissive" not in cargs # cuda.core gnu flags don't include it; bindings do + # Neither cuda.core nor cuda.bindings sets these gcc-only flags. + assert "-fpermissive" not in cargs assert "-fno-var-tracking-assignments" not in cargs @pytest.mark.agent_authored(model="grok-4.6") @@ -1018,6 +1018,30 @@ def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkey build_hooks._resolve_toolchain() assert os.environ["LDCXXSHARED"] == "clang++ -shared" + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_linux_opt_flag_set(self, monkeypatch): + """Linux opt build: -std=c++17, -O2; no gnu-only flags.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain(debug=False) + assert "-std=c++17" in cargs + assert "-g0" in cargs + assert "-O2" in cargs + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + assert "-Wl,--strip-all" in largs + + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_msvc_opt_flag_set(self, monkeypatch): + """MSVC opt build emits /std:c++17 and /O2.""" + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "/std:c++17" in cargs + assert "/O2" in cargs + class TestWithSccache: """_with_sccache: keep a leading sccache token, swap the compiler.""" From 8e06f2cbd168e5dd444a87b466c602c8523a33f7 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 08:05:29 -0700 Subject: [PATCH 3/9] fix: combine nested with statements in cython build_tests.py (SIM117) --- cuda_bindings/tests/cython/build_tests.py | 18 +++++++++--------- cuda_core/tests/cython/build_tests.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cuda_bindings/tests/cython/build_tests.py b/cuda_bindings/tests/cython/build_tests.py index 1ea736f6749..7fc609b1ec4 100644 --- a/cuda_bindings/tests/cython/build_tests.py +++ b/cuda_bindings/tests/cython/build_tests.py @@ -76,15 +76,15 @@ def _cythonize_tests(pyx_files): # .cython-bindings symlinks are not replaced. Relative aliases resolve # next to build_hooks.py (package root). stdlib_target = Path(Cython.__file__).parent / "Includes" - with build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib: - with build_hooks._stable_cython_alias( - _bindings_source_root(), Path(".cython-bindings-tests") - ) as rel_bindings: - return cythonize( - pyx_files, - include_path=[".", rel_bindings, rel_stdlib], - **cythonize_kwargs, - ) + with ( + build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, + build_hooks._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, + ): + return cythonize( + pyx_files, + include_path=[".", rel_bindings, rel_stdlib], + **cythonize_kwargs, + ) def main() -> None: diff --git a/cuda_core/tests/cython/build_tests.py b/cuda_core/tests/cython/build_tests.py index 6518e9a679b..292dd3cb815 100644 --- a/cuda_core/tests/cython/build_tests.py +++ b/cuda_core/tests/cython/build_tests.py @@ -75,15 +75,15 @@ def _cythonize_tests(pyx_files): # .cython-bindings symlinks are not replaced. Relative aliases resolve # next to build_hooks.py (package root). stdlib_target = Path(Cython.__file__).parent / "Includes" - with build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib: - with build_hooks._stable_cython_alias( - _bindings_source_root(), Path(".cython-bindings-tests") - ) as rel_bindings: - return cythonize( - pyx_files, - include_path=[".", rel_bindings, rel_stdlib], - **cythonize_kwargs, - ) + with ( + build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, + build_hooks._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, + ): + return cythonize( + pyx_files, + include_path=[".", rel_bindings, rel_stdlib], + **cythonize_kwargs, + ) def main() -> None: From c36d390a25b28581b50269d8f21f747177b8356e Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 08:26:17 -0700 Subject: [PATCH 4/9] build: share toolchain selection and flag assembly via _build_shared.py Add cuda_bindings/_build_shared.py as the single source of truth for the toolchain helpers (moved verbatim out of the duplicated block in both build_hooks.py files) and for the flag set unified in the previous commit. cuda_core/_build_shared.py is a symlink to it. Both packages already use backend-path = ["."], so the module is importable while the PEP 517 backend runs, and Python does not dereference symlinks in __file__, so Path(__file__)-relative locations still resolve under each package. resolve_toolchain() takes the per-package choices as arguments: a required cxx_std (no shared default; bindings stays on c++14, core on c++17), warnings_as_errors (core only, from CUDA_PYTHON_WERROR), and an optional tweak hook. Each build_hooks.py keeps a thin _resolve_toolchain() declaring just that. cuda-bindings moves -Wno-deprecated-declarations into its tweak. No behavior change: the resulting compiler/linker flags and CC/CXX/ LDCXXSHARED environment are identical across platform x toolchain x debug x coverage x werror, apart from -Wno-deprecated-declarations now being last for bindings. The Cython cache helpers stay in the synced block for now. Also include _build_shared.py in both sdists (MANIFEST.in), add it to known-first-party for isort, load it ahead of build_hooks.py in the tests and the tests/cython/build_tests.py scripts, and note the symlink in the package AGENTS.md files. --- cuda_bindings/AGENTS.md | 4 +- cuda_bindings/MANIFEST.in | 2 + cuda_bindings/_build_shared.py | 242 ++++++++++++++++++++++ cuda_bindings/build_hooks.py | 195 +++-------------- cuda_bindings/tests/cython/build_tests.py | 21 +- cuda_bindings/tests/test_build_hooks.py | 61 +++--- cuda_core/AGENTS.md | 5 +- cuda_core/MANIFEST.in | 2 + cuda_core/_build_shared.py | 1 + cuda_core/build_hooks.py | 197 ++---------------- cuda_core/tests/cython/build_tests.py | 21 +- cuda_core/tests/test_build_hooks.py | 66 +++--- ruff.toml | 4 +- toolshed/check_build_hooks_sync.py | 7 +- 14 files changed, 406 insertions(+), 422 deletions(-) create mode 100644 cuda_bindings/_build_shared.py create mode 120000 cuda_core/_build_shared.py diff --git a/cuda_bindings/AGENTS.md b/cuda_bindings/AGENTS.md index f0e4d9988c4..0f30fbb7154 100644 --- a/cuda_bindings/AGENTS.md +++ b/cuda_bindings/AGENTS.md @@ -19,7 +19,9 @@ subpackage in the `cuda-python` monorepo. - **Platform internals**: `cuda/bindings/_internal/` contains platform-specific implementation files and support code. - **Build backend**: `build_hooks.py` drives extension configuration and - Cythonization. + Cythonization. Logic shared with `cuda_core` (toolchain selection and the + compiler flag set) lives in `_build_shared.py`; `cuda_core/_build_shared.py` + is a symlink to this file, so an edit here changes both packages. ## Generated-source workflow diff --git a/cuda_bindings/MANIFEST.in b/cuda_bindings/MANIFEST.in index 4394e5d9fe3..6d01549d3e2 100644 --- a/cuda_bindings/MANIFEST.in +++ b/cuda_bindings/MANIFEST.in @@ -6,3 +6,5 @@ recursive-include cuda/ *.pyx *.pxd *.pxi *.pyx.in *.pxd.in *.pxi.in *.h # to the payload, causing file copying to the build environment failed exclude cuda/bindings cuda?bindings exclude cuda/bindings/_bindings cuda?bindings?_bindings +# canonical shared PEP 517 helper (cuda_core/_build_shared.py symlinks to the cuda_bindings copy) +include _build_shared.py diff --git a/cuda_bindings/_build_shared.py b/cuda_bindings/_build_shared.py new file mode 100644 index 00000000000..2d0440f65a3 --- /dev/null +++ b/cuda_bindings/_build_shared.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build helpers shared by the cuda-bindings and cuda-core PEP 517 backends. + +This is the single source of truth. ``cuda_core/_build_shared.py`` is a symlink +to this file. Python does not dereference symlinks in ``__file__``, so any +``Path(__file__)``-relative location in here resolves under whichever package +loads it. + +PEP 517 build isolation gives each backend its own ``build_hooks.py`` but not a +shared import path. Both packages declare ``backend-path = ["."]``, which puts +the package directory (and therefore this module) on ``sys.path`` while the +backend runs. + +Only what genuinely differs per package is parameterized, via the arguments of +``resolve_toolchain``: the C++ standard, warnings-as-errors, and an optional +``tweak`` hook for flags a single package needs. + +Note: There is no support guarantee for environment variables like +CUDA_PYTHON_TOOLCHAIN. They may be removed or changed in the future. +""" + +import os +import shlex +import shutil +import sys +import sysconfig + +# ----------------------------------------------------------------------- +# Toolchain selection + +_TOOLCHAINS_LINUX = ("gnu", "llvm") +_TOOLCHAINS_WINDOWS = ("msvc",) +_TOOLCHAIN_COMPILERS = { + "gnu": ("gcc", "g++"), + "llvm": ("clang", "clang++"), + "msvc": (None, None), +} + + +def _resolve_toolchain_name(): + """Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx). + + The default toolchain (gnu on Linux, msvc on Windows) is the first entry + of the platform's allowed tuple. cc/cxx are the compiler binaries for the + toolchain (None for msvc, which distutils discovers via the MSVC env). + """ + if sys.platform == "win32": + platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS + else: + platform_key, allowed = "linux", _TOOLCHAINS_LINUX + name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower() + if name not in allowed: + raise RuntimeError( + f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}." + ) + cc, cxx = _TOOLCHAIN_COMPILERS[name] + explicit = bool(os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip()) + return name, allowed, cc, cxx, explicit + + +def _with_compiler(command, compiler): + """Replace the leading compiler on a linker command; keep flags. + + Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat + -shared ...``. Only the executable changes so those flags stay on the + link line. The command is tokenized with shlex so quoted arguments + survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are + not rewritten this way: they may already be a launcher plus compiler + (``sccache cc``). + """ + if not command or not command.strip(): + return compiler + parts = shlex.split(command) + # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it + # off before it substitutes the compiler, so it still reaches the link line. + prefix_end = 0 + if parts and os.path.basename(parts[0]) == "env": + prefix_end = 1 + # Match setuptools' _split_env: any token with ``=`` is an env operand + # (covers both ``VAR=value`` and ``--unset=VAR`` long options). + while prefix_end < len(parts) and "=" in parts[prefix_end]: + prefix_end += 1 + # Everything else before the first flag is the old compiler (or a launcher + # for it; setuptools takes the launcher from CXX instead). + i = prefix_end + while i < len(parts) and not parts[i].startswith("-"): + i += 1 + return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) + + +def _with_sccache(current, compiler): + """Keep a leading sccache token when the toolchain picks a compiler. + + CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit + toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. + """ + if current: + launcher = current.split()[0] + if os.path.basename(launcher) == "sccache": + return f"{launcher} {compiler}" + return compiler + + +def _apply_toolchain_env(cc, cxx, explicit): + """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. + + The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally + does not touch the env, so an externally-set compiler (e.g. + CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN + override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing + sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). + Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the + environment if set there and from sysconfig otherwise; only the compiler + is swapped. LDSHARED is left unset so distutils rewrites it from CC. + """ + if explicit and cc is not None: + os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) + os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) + # An LDCXXSHARED the user already exported takes precedence over + # sysconfig's, as CC/CXX do; either way only the compiler is swapped. + ldcxxshared = ( + os.environ.get("LDCXXSHARED") + or sysconfig.get_config_var("LDCXXSHARED") + or sysconfig.get_config_var("LDSHARED") + ) + os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" + + +def _check_toolchain_available(name): + """Preflight: verify the selected toolchain's tools are on PATH. + + No-op for the platform default (distutils discovers those). For llvm, + probes clang, clang++, and ld.lld so a missing toolchain fails fast with a + helpful message instead of a cryptic compile error. + """ + if name != "llvm": + return + tools = ("clang", "clang++", "ld.lld") + missing = [t for t in tools if shutil.which(t) is None] + if missing: + raise RuntimeError( + f"CUDA_PYTHON_TOOLCHAIN=llvm but required tool(s) not found on PATH: " + f"{', '.join(missing)}. Install clang and lld " + f"(e.g. `apt install clang lld` or `dnf install clang lld`) " + f"or set CUDA_PYTHON_TOOLCHAIN=gnu." + ) + + +# ----------------------------------------------------------------------- +# Compiler and linker flags + + +def _build_flags(name, cxx_std, debug, coverage, warnings_as_errors): + """Return (extra_compile_args, extra_link_args) for toolchain ``name``. + + The one flag set used by both packages: + + - Linux compile: ``-std=c++{cxx_std}``, then ``-g0 -O2`` (opt) or + ``-g -O0 -D _GLIBCXX_ASSERTIONS`` (debug). + - Linux link: ``-fuse-ld=lld`` (llvm) and ``-Wl,--strip-all`` (opt). + - MSVC compile: ``/std:c++{cxx_std}`` and ``/O2``. Modern setuptools no + longer forces ``/Ox``, so the optimization level is set explicitly for + symmetry with Linux ``-O2``. Debug builds are not supported on Windows. + - Coverage: Cython tracing defines. + - Warnings-as-errors: ``-Werror`` (Linux) or ``/WX`` (MSVC). + """ + extra_compile_args = [] + extra_link_args = [] + + if name == "msvc": + if debug: + raise RuntimeError("Debuggable builds are not supported on Windows.") + extra_compile_args += [f"/std:c++{cxx_std}", "/O2"] + else: + # Common Linux compile flags. + extra_compile_args += [f"-std=c++{cxx_std}"] + # Compiler-specific flags. + if name == "llvm": + extra_link_args += ["-fuse-ld=lld"] + # Common Linux debug/opt flags. + if debug: + extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] + else: + extra_compile_args += ["-g0", "-O2"] + extra_link_args += ["-Wl,--strip-all"] + + if coverage: + # CYTHON_TRACE_NOGIL indicates to trace nogil functions. It is not + # related to free-threading builds. + extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] + + if warnings_as_errors: + # The MSVC exemptions cover warnings that Cython's utility code + # produces in every module and the .pyx sources cannot fix: + # - C4551 ("function call missing argument list"), hundreds per + # module. + # - C4244 (narrowing): the overflow-check helpers that + # @cython.overflowcheck(True) instantiates for _layout.pxd narrow + # int64 to int inside Cython's own code. + # gcc and clang need no exemption. The one generated warning they + # report in cuda.core, the unused @overload wrappers of + # Graph.__getitem__, is silenced by a pragma in + # cuda/core/graph/_graph_builder.pyx. + if name == "msvc": + extra_compile_args += ["/WX", "/wd4551", "/wd4244"] + else: + extra_compile_args += ["-Werror"] + + return extra_compile_args, extra_link_args + + +def resolve_toolchain(*, cxx_std, debug=False, compile_for_coverage=False, warnings_as_errors=False, tweak=None): + """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN. + + Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default + toolchain (gnu on Linux, msvc on Windows) reproduces the previous build + behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler + (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on + Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' + customize_compiler picks them up. + + The per-package choices are arguments: + + - ``cxx_std`` (required): the C++ standard, e.g. ``14``. There is + deliberately no shared default; each package picks its own. + - ``warnings_as_errors``: opt in to ``-Werror`` / ``/WX``. + - ``tweak``: optional callable ``tweak(name, extra_compile_args, + extra_link_args)`` returning the adjusted ``(extra_compile_args, + extra_link_args)`` pair, for the few flags a single package needs that + do not belong in the shared set. + """ + name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() + + extra_compile_args, extra_link_args = _build_flags(name, cxx_std, debug, compile_for_coverage, warnings_as_errors) + if tweak is not None: + extra_compile_args, extra_link_args = tweak(name, extra_compile_args, extra_link_args) + + _apply_toolchain_env(cc, cxx, explicit) + + return name, cc, cxx, extra_compile_args, extra_link_args diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index eab8f7c6299..08a1a2344c3 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -14,7 +14,6 @@ import hashlib import os import re -import shlex import shutil import sys import sysconfig @@ -26,6 +25,8 @@ from setuptools import build_meta as _build_meta from setuptools.extension import Extension +from _build_shared import _check_toolchain_available, _resolve_toolchain_name, resolve_toolchain + # Metadata hooks delegate directly to setuptools -- no CUDA needed. prepare_metadata_for_build_editable = _build_meta.prepare_metadata_for_build_editable prepare_metadata_for_build_wheel = _build_meta.prepare_metadata_for_build_wheel @@ -165,134 +166,16 @@ def _check_cuda_headers(cuda_path: str) -> None: # ----------------------------------------------------------------------- -# Toolchain selection +# Cython cache helpers # -# There is one shared helper block below, duplicated verbatim in -# cuda_core/build_hooks.py (keep it in sync; enforced by -# toolshed/check_build_hooks_sync.py). It contains the toolchain helpers and -# the Cython cache helpers. Only the per-package _resolve_toolchain() flag -# assembly that follows the shared block is package-specific (it differs -# because the two packages use different C++ standards and opt levels). - -# --- begin shared build helpers (keep in sync) --- -_TOOLCHAINS_LINUX = ("gnu", "llvm") -_TOOLCHAINS_WINDOWS = ("msvc",) -_TOOLCHAIN_COMPILERS = { - "gnu": ("gcc", "g++"), - "llvm": ("clang", "clang++"), - "msvc": (None, None), -} - - -def _resolve_toolchain_name(): - """Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx). - - The default toolchain (gnu on Linux, msvc on Windows) is the first entry - of the platform's allowed tuple. cc/cxx are the compiler binaries for the - toolchain (None for msvc, which distutils discovers via the MSVC env). - """ - if sys.platform == "win32": - platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS - else: - platform_key, allowed = "linux", _TOOLCHAINS_LINUX - name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower() - if name not in allowed: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}." - ) - cc, cxx = _TOOLCHAIN_COMPILERS[name] - explicit = bool(os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip()) - return name, allowed, cc, cxx, explicit - - -def _with_compiler(command, compiler): - """Replace the leading compiler on a linker command; keep flags. - - Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat - -shared ...``. Only the executable changes so those flags stay on the - link line. The command is tokenized with shlex so quoted arguments - survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are - not rewritten this way: they may already be a launcher plus compiler - (``sccache cc``). - """ - if not command or not command.strip(): - return compiler - parts = shlex.split(command) - # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it - # off before it substitutes the compiler, so it still reaches the link line. - prefix_end = 0 - if parts and os.path.basename(parts[0]) == "env": - prefix_end = 1 - # Match setuptools' _split_env: any token with ``=`` is an env operand - # (covers both ``VAR=value`` and ``--unset=VAR`` long options). - while prefix_end < len(parts) and "=" in parts[prefix_end]: - prefix_end += 1 - # Everything else before the first flag is the old compiler (or a launcher - # for it; setuptools takes the launcher from CXX instead). - i = prefix_end - while i < len(parts) and not parts[i].startswith("-"): - i += 1 - return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) - - -def _with_sccache(current, compiler): - """Keep a leading sccache token when the toolchain picks a compiler. - - CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit - toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. - """ - if current: - launcher = current.split()[0] - if os.path.basename(launcher) == "sccache": - return f"{launcher} {compiler}" - return compiler - - -def _apply_toolchain_env(cc, cxx, explicit): - """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. - - The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally - does not touch the env, so an externally-set compiler (e.g. - CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN - override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing - sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). - Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the - environment if set there and from sysconfig otherwise; only the compiler - is swapped. LDSHARED is left unset so distutils rewrites it from CC. - """ - if explicit and cc is not None: - os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) - os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) - # An LDCXXSHARED the user already exported takes precedence over - # sysconfig's, as CC/CXX do; either way only the compiler is swapped. - ldcxxshared = ( - os.environ.get("LDCXXSHARED") - or sysconfig.get_config_var("LDCXXSHARED") - or sysconfig.get_config_var("LDSHARED") - ) - os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" - - -def _check_toolchain_available(name): - """Preflight: verify the selected toolchain's tools are on PATH. - - No-op for the platform default (distutils discovers those). For llvm, - probes clang, clang++, and ld.lld so a missing toolchain fails fast with a - helpful message instead of a cryptic compile error. - """ - if name != "llvm": - return - tools = ("clang", "clang++", "ld.lld") - missing = [t for t in tools if shutil.which(t) is None] - if missing: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN=llvm but required tool(s) not found on PATH: " - f"{', '.join(missing)}. Install clang and lld " - f"(e.g. `apt install clang lld` or `dnf install clang lld`) " - f"or set CUDA_PYTHON_TOOLCHAIN=gnu." - ) +# The toolchain helpers and flag assembly live in _build_shared.py (the +# single source of truth; cuda_core/_build_shared.py is a symlink to it). +# The helper block below is still duplicated verbatim in +# cuda_{bindings,core}/build_hooks.py (keep it in sync; enforced by +# toolshed/check_build_hooks_sync.py). +# --- begin shared build helpers (keep in sync) --- # === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === # Workaround for Cython issue #7532: Cython's native cache fingerprint omits # `compiler_directives`, so builds with different directives (e.g. linetrace @@ -421,54 +304,30 @@ def _stable_cython_alias(target: Path, alias: Path): # --- end shared build helpers --- -def _resolve_toolchain(debug=False, compile_for_coverage=False): - """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN. - - Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default - toolchain (gnu on Linux, msvc on Windows) reproduces the previous build - behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler - (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on - Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' - customize_compiler picks them up. - """ - name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() - - extra_compile_args = [] - extra_link_args = [] - - if name == "msvc": - # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args - # from gcc's c++17 variadic-template expansion. - extra_compile_args += ["/std:c++14", "/O2"] - if debug: - raise RuntimeError("Debuggable builds are not supported on Windows.") - else: - # Common Linux compile flags. - # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args - # from gcc's c++17 variadic-template expansion. - extra_compile_args += ["-std=c++14"] +def _tweak_flags(name, extra_compile_args, extra_link_args): + """cuda-bindings flags that do not belong in the shared set.""" + if name != "msvc": # cudaMemcpy*Array* and cudaGetDriverEntryPoint are deprecated but still # supported; suppress the resulting warnings so a future -Werror build # is not broken by Cython-generated calls we cannot control. - extra_compile_args += ["-Wno-deprecated-declarations"] - # Compiler-specific flags. - if name == "llvm": - extra_link_args += ["-fuse-ld=lld"] - # Common Linux debug/opt flags. - if debug: - extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] - else: - extra_compile_args += ["-g0", "-O2"] - extra_link_args += ["-Wl,--strip-all"] + extra_compile_args = [*extra_compile_args, "-Wno-deprecated-declarations"] + return extra_compile_args, extra_link_args - if compile_for_coverage: - # CYTHON_TRACE_NOGIL indicates to trace nogil functions. It is not - # related to free-threading builds. - extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] - _apply_toolchain_env(cc, cxx, explicit) +def _resolve_toolchain(debug=False, compile_for_coverage=False): + """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN (cuda.bindings flags). - return name, cc, cxx, extra_compile_args, extra_link_args + See _build_shared.resolve_toolchain() for the return value and the + environment handling. What is specific to cuda.bindings is declared here. + """ + return resolve_toolchain( + # c++14: raising to c++17 costs a measured ~15% on launch_{256,512}_args + # from gcc's c++17 variadic-template expansion. + cxx_std=14, + debug=debug, + compile_for_coverage=compile_for_coverage, + tweak=_tweak_flags, + ) # ----------------------------------------------------------------------- diff --git a/cuda_bindings/tests/cython/build_tests.py b/cuda_bindings/tests/cython/build_tests.py index 7fc609b1ec4..9943f74f66d 100644 --- a/cuda_bindings/tests/cython/build_tests.py +++ b/cuda_bindings/tests/cython/build_tests.py @@ -29,17 +29,26 @@ _COMPILER_DIRECTIVES = {"freethreading_compatible": True} -def _load_build_hooks(): - # PEP 517 backend, not an installed module. Load by path so we do not put - # cuda_bindings/ on sys.path (that would shadow the installed package). - build_hooks_path = Path(__file__).resolve().parents[2] / "build_hooks.py" - spec = importlib.util.spec_from_file_location("cuda_bindings_build_hooks", build_hooks_path) +def _load_module(name, path, *, register=False): + # PEP 517 backend files, not installed modules. Load by path so we do not + # put the package directory on sys.path (that would shadow the installed + # package). With ``register`` the module is also entered into sys.modules, + # which is how build_hooks.py's ``from _build_shared import ...`` finds it. + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + if register: + sys.modules[name] = module spec.loader.exec_module(module) return module -build_hooks = _load_build_hooks() +def _load_build_hooks(modname): + package_root = Path(__file__).resolve().parents[2] + _load_module("_build_shared", package_root / "_build_shared.py", register=True) + return _load_module(modname, package_root / "build_hooks.py") + + +build_hooks = _load_build_hooks("cuda_bindings_build_hooks") def _bindings_source_root() -> Path: diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index b7606f997ba..4f522c5b8f6 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -31,16 +31,28 @@ from setuptools._distutils.sysconfig import customize_compiler -def _load_build_hooks(): - """Load build_hooks module from source without polluting sys.path.""" - build_hooks_path = Path(__file__).parent.parent / "build_hooks.py" - spec = importlib.util.spec_from_file_location("build_hooks", build_hooks_path) +def _load_module(name, path, *, register=False): + """Load a module from source without permanently modifying sys.path. + + build_hooks.py and _build_shared.py are PEP 517 backend files, not + installed modules. We use importlib to load them directly from source to + avoid polluting sys.path with the package directory (which contains + cuda/ source that could shadow the installed package). With ``register`` + the module is also entered into sys.modules, which is how build_hooks.py's + ``from _build_shared import ...`` finds this copy. + """ + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + if register: + sys.modules[name] = module spec.loader.exec_module(module) return module -build_hooks = _load_build_hooks() +# Load the modules once at import time; _build_shared must come first. +_PACKAGE_ROOT = Path(__file__).parent.parent +_build_shared = _load_module("_build_shared", _PACKAGE_ROOT / "_build_shared.py", register=True) +build_hooks = _load_module("build_hooks", _PACKAGE_ROOT / "build_hooks.py") @pytest.fixture(autouse=True) @@ -221,19 +233,20 @@ class TestWithSccache: @pytest.mark.agent_authored(model="grok-4.6") def test_keeps_sccache_and_swaps_compiler(self): - assert build_hooks._with_sccache("sccache cc", "clang") == "sccache clang" + assert _build_shared._with_sccache("sccache cc", "clang") == "sccache clang" @pytest.mark.agent_authored(model="grok-4.6") def test_keeps_absolute_sccache_path(self): assert ( - build_hooks._with_sccache("/host/usr/local/bin/sccache cc", "clang") == "/host/usr/local/bin/sccache clang" + _build_shared._with_sccache("/host/usr/local/bin/sccache cc", "clang") + == "/host/usr/local/bin/sccache clang" ) @pytest.mark.agent_authored(model="grok-4.6") def test_bare_or_unrelated_cc_returns_compiler(self): - assert build_hooks._with_sccache("", "clang") == "clang" - assert build_hooks._with_sccache("gcc", "clang") == "clang" - assert build_hooks._with_sccache("ccache gcc", "clang") == "clang" + assert _build_shared._with_sccache("", "clang") == "clang" + assert _build_shared._with_sccache("gcc", "clang") == "clang" + assert _build_shared._with_sccache("ccache gcc", "clang") == "clang" class TestWithCompiler: @@ -242,36 +255,36 @@ class TestWithCompiler: @pytest.mark.agent_authored(model="grok-4.6") def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): assert ( - build_hooks._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") + _build_shared._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" ) @pytest.mark.agent_authored(model="grok-4.6") def test_compiler_only_command_returns_compiler(self): - assert build_hooks._with_compiler("g++", "clang++") == "clang++" - assert build_hooks._with_compiler("", "clang++") == "clang++" - assert build_hooks._with_compiler(None, "clang++") == "clang++" + assert _build_shared._with_compiler("g++", "clang++") == "clang++" + assert _build_shared._with_compiler("", "clang++") == "clang++" + assert _build_shared._with_compiler(None, "clang++") == "clang++" @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_keeps_quoted_arguments_intact(self): - result = build_hooks._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") + result = _build_shared._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_drops_launcher_before_compiler(self): # setuptools takes the launcher from CXX; keeping a second copy here would # leave a stray compiler argument on the link line. - assert build_hooks._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" + assert _build_shared._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_keeps_env_prefix(self): assert ( - build_hooks._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") + _build_shared._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") == "env LIBRARY_PATH=/custom/lib clang++ -shared" ) # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env assert ( - build_hooks._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") + _build_shared._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") == "env --unset=LD_LIBRARY_PATH clang++ -shared" ) @@ -356,22 +369,22 @@ class TestCheckToolchainAvailable: @pytest.mark.agent_authored(model="glm-5.2") def test_default_is_noop(self): - build_hooks._check_toolchain_available("gnu") - build_hooks._check_toolchain_available("msvc") + _build_shared._check_toolchain_available("gnu") + _build_shared._check_toolchain_available("msvc") @pytest.mark.agent_authored(model="glm-5.2") def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): def fake_which(name): return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name - monkeypatch.setattr(build_hooks.shutil, "which", fake_which) + monkeypatch.setattr(_build_shared.shutil, "which", fake_which) with pytest.raises(RuntimeError, match="clang and lld"): - build_hooks._check_toolchain_available("llvm") + _build_shared._check_toolchain_available("llvm") @pytest.mark.agent_authored(model="glm-5.2") def test_llvm_present_passes(self, monkeypatch): - monkeypatch.setattr(build_hooks.shutil, "which", lambda name: "/bin/" + name) - build_hooks._check_toolchain_available("llvm") + monkeypatch.setattr(_build_shared.shutil, "which", lambda name: "/bin/" + name) + _build_shared._check_toolchain_available("llvm") @pytest.fixture diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 4053d48958a..fac6d1b2a85 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -23,7 +23,10 @@ This file describes `cuda_core`, the high-level Pythonic CUDA subpackage in the one `_cpp/.cpp` or as a directory `_cpp//` whose sources all compile into the `_` extension (`_cpp/rt/` for `_rt`). - **Build backend**: `build_hooks.py` handles Cython extension setup and build - dependency wiring. + dependency wiring. Logic shared with `cuda_bindings` (toolchain selection and + the compiler flag set) lives in `_build_shared.py`, a symlink to + `cuda_bindings/_build_shared.py`; an edit through either path changes both + packages. ## Build and version coupling diff --git a/cuda_core/MANIFEST.in b/cuda_core/MANIFEST.in index f63b324cab7..b4129a4166b 100644 --- a/cuda_core/MANIFEST.in +++ b/cuda_core/MANIFEST.in @@ -7,3 +7,5 @@ recursive-include cuda/core/_cpp *.cpp *.h *.hpp recursive-include cuda/core/_include *.h *.hpp include cuda/core/py.typed include NOTICE +# canonical shared PEP 517 helper (cuda_core/_build_shared.py symlinks to the cuda_bindings copy) +include _build_shared.py diff --git a/cuda_core/_build_shared.py b/cuda_core/_build_shared.py new file mode 120000 index 00000000000..31696363b94 --- /dev/null +++ b/cuda_core/_build_shared.py @@ -0,0 +1 @@ +../cuda_bindings/_build_shared.py \ No newline at end of file diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index cf09a1bd288..009825b51f6 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -14,8 +14,6 @@ import importlib.util import os import re -import shlex -import shutil import sys import sysconfig import tempfile @@ -30,6 +28,8 @@ from setuptools import Extension from setuptools import build_meta as _build_meta +from _build_shared import _check_toolchain_available, resolve_toolchain + prepare_metadata_for_build_editable = _build_meta.prepare_metadata_for_build_editable prepare_metadata_for_build_wheel = _build_meta.prepare_metadata_for_build_wheel build_sdist = _build_meta.build_sdist @@ -212,134 +212,16 @@ def _read_cuda_h_version(cuda_path: str) -> int: # ----------------------------------------------------------------------- -# Toolchain selection +# Cython cache helpers # -# There is one shared helper block below, duplicated verbatim in -# cuda_bindings/build_hooks.py (keep it in sync; enforced by -# toolshed/check_build_hooks_sync.py). It contains the toolchain helpers and -# the Cython cache helpers. Only the per-package _resolve_toolchain() flag -# assembly that follows the shared block is package-specific (it differs -# because the two packages use different C++ standards and opt levels). - -# --- begin shared build helpers (keep in sync) --- -_TOOLCHAINS_LINUX = ("gnu", "llvm") -_TOOLCHAINS_WINDOWS = ("msvc",) -_TOOLCHAIN_COMPILERS = { - "gnu": ("gcc", "g++"), - "llvm": ("clang", "clang++"), - "msvc": (None, None), -} - - -def _resolve_toolchain_name(): - """Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx). - - The default toolchain (gnu on Linux, msvc on Windows) is the first entry - of the platform's allowed tuple. cc/cxx are the compiler binaries for the - toolchain (None for msvc, which distutils discovers via the MSVC env). - """ - if sys.platform == "win32": - platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS - else: - platform_key, allowed = "linux", _TOOLCHAINS_LINUX - name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower() - if name not in allowed: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}." - ) - cc, cxx = _TOOLCHAIN_COMPILERS[name] - explicit = bool(os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip()) - return name, allowed, cc, cxx, explicit - - -def _with_compiler(command, compiler): - """Replace the leading compiler on a linker command; keep flags. - - Conda ``LDCXXSHARED`` looks like ``g++ -pthread -B .../python_compiler_compat - -shared ...``. Only the executable changes so those flags stay on the - link line. The command is tokenized with shlex so quoted arguments - survive, and a leading ``env VAR=value`` prefix is preserved. CC/CXX are - not rewritten this way: they may already be a launcher plus compiler - (``sccache cc``). - """ - if not command or not command.strip(): - return compiler - parts = shlex.split(command) - # Keep an ``env VAR=value ...`` prefix: setuptools' C++ link step splits it - # off before it substitutes the compiler, so it still reaches the link line. - prefix_end = 0 - if parts and os.path.basename(parts[0]) == "env": - prefix_end = 1 - # Match setuptools' _split_env: any token with ``=`` is an env operand - # (covers both ``VAR=value`` and ``--unset=VAR`` long options). - while prefix_end < len(parts) and "=" in parts[prefix_end]: - prefix_end += 1 - # Everything else before the first flag is the old compiler (or a launcher - # for it; setuptools takes the launcher from CXX instead). - i = prefix_end - while i < len(parts) and not parts[i].startswith("-"): - i += 1 - return shlex.join([*parts[:prefix_end], compiler, *parts[i:]]) - - -def _with_sccache(current, compiler): - """Keep a leading sccache token when the toolchain picks a compiler. - - CI sets ``CC="sccache cc"`` or ``CC="/host/.../sccache cc"``. An explicit - toolchain then becomes ``CC="sccache clang"`` rather than a bare compiler. - """ - if current: - launcher = current.split()[0] - if os.path.basename(launcher) == "sccache": - return f"{launcher} {compiler}" - return compiler - - -def _apply_toolchain_env(cc, cxx, explicit): - """Set CC/CXX/LDCXXSHARED for an explicitly-chosen toolchain. - - The default path (CUDA_PYTHON_TOOLCHAIN unset) intentionally - does not touch the env, so an externally-set compiler (e.g. - CC="sccache cc" in CI) keeps working. An explicit CUDA_PYTHON_TOOLCHAIN - override (incl. =gnu) sets CC/CXX to the toolchain compiler; an existing - sccache prefix is kept (CC="sccache cc" + llvm -> CC="sccache clang"). - Extras on LDCXXSHARED (rpath, -pthread, -B, ...) are kept, taken from the - environment if set there and from sysconfig otherwise; only the compiler - is swapped. LDSHARED is left unset so distutils rewrites it from CC. - """ - if explicit and cc is not None: - os.environ["CC"] = _with_sccache(os.environ.get("CC", ""), cc) - os.environ["CXX"] = _with_sccache(os.environ.get("CXX", ""), cxx) - # An LDCXXSHARED the user already exported takes precedence over - # sysconfig's, as CC/CXX do; either way only the compiler is swapped. - ldcxxshared = ( - os.environ.get("LDCXXSHARED") - or sysconfig.get_config_var("LDCXXSHARED") - or sysconfig.get_config_var("LDSHARED") - ) - os.environ["LDCXXSHARED"] = _with_compiler(ldcxxshared, cxx) if ldcxxshared else f"{cxx} -shared" - - -def _check_toolchain_available(name): - """Preflight: verify the selected toolchain's tools are on PATH. - - No-op for the platform default (distutils discovers those). For llvm, - probes clang, clang++, and ld.lld so a missing toolchain fails fast with a - helpful message instead of a cryptic compile error. - """ - if name != "llvm": - return - tools = ("clang", "clang++", "ld.lld") - missing = [t for t in tools if shutil.which(t) is None] - if missing: - raise RuntimeError( - f"CUDA_PYTHON_TOOLCHAIN=llvm but required tool(s) not found on PATH: " - f"{', '.join(missing)}. Install clang and lld " - f"(e.g. `apt install clang lld` or `dnf install clang lld`) " - f"or set CUDA_PYTHON_TOOLCHAIN=gnu." - ) +# The toolchain helpers and flag assembly live in _build_shared.py (the +# single source of truth; cuda_core/_build_shared.py is a symlink to it). +# The helper block below is still duplicated verbatim in +# cuda_{bindings,core}/build_hooks.py (keep it in sync; enforced by +# toolshed/check_build_hooks_sync.py). +# --- begin shared build helpers (keep in sync) --- # === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === # Workaround for Cython issue #7532: Cython's native cache fingerprint omits # `compiler_directives`, so builds with different directives (e.g. linetrace @@ -471,61 +353,16 @@ def _stable_cython_alias(target: Path, alias: Path): def _resolve_toolchain(debug=False, compile_for_coverage=False): """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN (cuda.core flags). - Returns (name, cc, cxx, extra_compile_args, extra_link_args). The default - toolchain (gnu on Linux, msvc on Windows) reproduces the previous build - behavior and does not touch CC/CXX/LDCXXSHARED, so an externally-set compiler - (e.g. CC="sccache cc") keeps working. A non-default toolchain (llvm on - Linux) selects clang/clang++ and lld and sets CC/CXX/LDCXXSHARED so distutils' - customize_compiler picks them up. + See _build_shared.resolve_toolchain() for the return value and the + environment handling. What is specific to cuda.core is declared here. """ - name, _allowed, cc, cxx, explicit = _resolve_toolchain_name() - - extra_compile_args = [] - extra_link_args = [] - - if name == "msvc": - # c++17: required by structured bindings and if constexpr in cuda/core/_cpp/. - extra_compile_args += ["/std:c++17", "/O2"] - if debug: - raise RuntimeError("Debuggable builds are not supported on Windows.") - else: - # Common Linux compile flags. + return resolve_toolchain( # c++17: required by structured bindings and if constexpr in cuda/core/_cpp/. - extra_compile_args += ["-std=c++17"] - # Compiler-specific flags. - if name == "llvm": - extra_link_args += ["-fuse-ld=lld"] - # Common Linux debug/opt flags. - if debug: - extra_compile_args += ["-g", "-O0", "-D _GLIBCXX_ASSERTIONS"] - else: - extra_compile_args += ["-g0", "-O2"] - extra_link_args += ["-Wl,--strip-all"] - - if compile_for_coverage: - # CYTHON_TRACE_NOGIL indicates to trace nogil functions. It is not - # related to free-threading builds. - extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] - - if WARNINGS_AS_ERRORS: - # The MSVC exemptions cover warnings that Cython's utility code - # produces in every module and the .pyx sources cannot fix: - # - C4551 ("function call missing argument list"), hundreds per - # module. - # - C4244 (narrowing): the overflow-check helpers that - # @cython.overflowcheck(True) instantiates for _layout.pxd narrow - # int64 to int inside Cython's own code. - # gcc and clang need no exemption. The one generated warning they - # report, the unused @overload wrappers of Graph.__getitem__, is - # silenced by a pragma in cuda/core/graph/_graph_builder.pyx. - if name == "msvc": - extra_compile_args += ["/WX", "/wd4551", "/wd4244"] - else: - extra_compile_args += ["-Werror"] - - _apply_toolchain_env(cc, cxx, explicit) - - return name, cc, cxx, extra_compile_args, extra_link_args + cxx_std=17, + debug=debug, + compile_for_coverage=compile_for_coverage, + warnings_as_errors=WARNINGS_AS_ERRORS, + ) @functools.cache diff --git a/cuda_core/tests/cython/build_tests.py b/cuda_core/tests/cython/build_tests.py index 292dd3cb815..453cf3c7915 100644 --- a/cuda_core/tests/cython/build_tests.py +++ b/cuda_core/tests/cython/build_tests.py @@ -29,17 +29,26 @@ _COMPILER_DIRECTIVES = {"freethreading_compatible": True} -def _load_build_hooks(): - # PEP 517 backend, not an installed module. Load by path so we do not put - # cuda_core/ on sys.path (that would shadow the installed package). - build_hooks_path = Path(__file__).resolve().parents[2] / "build_hooks.py" - spec = importlib.util.spec_from_file_location("cuda_core_build_hooks", build_hooks_path) +def _load_module(name, path, *, register=False): + # PEP 517 backend files, not installed modules. Load by path so we do not + # put the package directory on sys.path (that would shadow the installed + # package). With ``register`` the module is also entered into sys.modules, + # which is how build_hooks.py's ``from _build_shared import ...`` finds it. + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + if register: + sys.modules[name] = module spec.loader.exec_module(module) return module -build_hooks = _load_build_hooks() +def _load_build_hooks(modname): + package_root = Path(__file__).resolve().parents[2] + _load_module("_build_shared", package_root / "_build_shared.py", register=True) + return _load_module(modname, package_root / "build_hooks.py") + + +build_hooks = _load_build_hooks("cuda_core_build_hooks") def _bindings_source_root() -> Path: diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index f5a431c9810..9883da3c26b 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -41,23 +41,28 @@ from cuda.pathfinder import get_cuda_path_or_home -def _load_build_hooks(): - """Load build_hooks module from source without permanently modifying sys.path. - - build_hooks.py is a PEP 517 build backend, not an installed module. - We use importlib to load it directly from source to avoid polluting - sys.path with the cuda_core/ directory (which contains cuda/core/ source - that could shadow the installed package). +def _load_module(name, path, *, register=False): + """Load a module from source without permanently modifying sys.path. + + build_hooks.py and _build_shared.py are PEP 517 backend files, not + installed modules. We use importlib to load them directly from source to + avoid polluting sys.path with the package directory (which contains + cuda/ source that could shadow the installed package). With ``register`` + the module is also entered into sys.modules, which is how build_hooks.py's + ``from _build_shared import ...`` finds this copy. """ - build_hooks_path = Path(__file__).parent.parent / "build_hooks.py" - spec = importlib.util.spec_from_file_location("build_hooks", build_hooks_path) + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) + if register: + sys.modules[name] = module spec.loader.exec_module(module) return module -# Load the module once at import time -build_hooks = _load_build_hooks() +# Load the modules once at import time; _build_shared must come first. +_PACKAGE_ROOT = Path(__file__).parent.parent +_build_shared = _load_module("_build_shared", _PACKAGE_ROOT / "_build_shared.py", register=True) +build_hooks = _load_module("build_hooks", _PACKAGE_ROOT / "build_hooks.py") @pytest.fixture(autouse=True) @@ -1048,19 +1053,20 @@ class TestWithSccache: @pytest.mark.agent_authored(model="grok-4.6") def test_keeps_sccache_and_swaps_compiler(self): - assert build_hooks._with_sccache("sccache cc", "clang") == "sccache clang" + assert _build_shared._with_sccache("sccache cc", "clang") == "sccache clang" @pytest.mark.agent_authored(model="grok-4.6") def test_keeps_absolute_sccache_path(self): assert ( - build_hooks._with_sccache("/host/usr/local/bin/sccache cc", "clang") == "/host/usr/local/bin/sccache clang" + _build_shared._with_sccache("/host/usr/local/bin/sccache cc", "clang") + == "/host/usr/local/bin/sccache clang" ) @pytest.mark.agent_authored(model="grok-4.6") def test_bare_or_unrelated_cc_returns_compiler(self): - assert build_hooks._with_sccache("", "clang") == "clang" - assert build_hooks._with_sccache("gcc", "clang") == "clang" - assert build_hooks._with_sccache("ccache gcc", "clang") == "clang" + assert _build_shared._with_sccache("", "clang") == "clang" + assert _build_shared._with_sccache("gcc", "clang") == "clang" + assert _build_shared._with_sccache("ccache gcc", "clang") == "clang" class TestWithCompiler: @@ -1069,36 +1075,36 @@ class TestWithCompiler: @pytest.mark.agent_authored(model="grok-4.6") def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): assert ( - build_hooks._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") + _build_shared._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" ) @pytest.mark.agent_authored(model="grok-4.6") def test_compiler_only_command_returns_compiler(self): - assert build_hooks._with_compiler("g++", "clang++") == "clang++" - assert build_hooks._with_compiler("", "clang++") == "clang++" - assert build_hooks._with_compiler(None, "clang++") == "clang++" + assert _build_shared._with_compiler("g++", "clang++") == "clang++" + assert _build_shared._with_compiler("", "clang++") == "clang++" + assert _build_shared._with_compiler(None, "clang++") == "clang++" @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_keeps_quoted_arguments_intact(self): - result = build_hooks._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") + result = _build_shared._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_drops_launcher_before_compiler(self): # setuptools takes the launcher from CXX; keeping a second copy here would # leave a stray compiler argument on the link line. - assert build_hooks._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" + assert _build_shared._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" @pytest.mark.agent_authored(model="claude-sonnet-5.5") def test_keeps_env_prefix(self): assert ( - build_hooks._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") + _build_shared._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") == "env LIBRARY_PATH=/custom/lib clang++ -shared" ) # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env assert ( - build_hooks._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") + _build_shared._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") == "env --unset=LD_LIBRARY_PATH clang++ -shared" ) @@ -1184,22 +1190,22 @@ class TestCheckToolchainAvailable: @pytest.mark.agent_authored(model="glm-5.2") def test_default_is_noop(self): # The platform default never preflights. - build_hooks._check_toolchain_available("gnu") - build_hooks._check_toolchain_available("msvc") + _build_shared._check_toolchain_available("gnu") + _build_shared._check_toolchain_available("msvc") @pytest.mark.agent_authored(model="glm-5.2") def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): def fake_which(name): return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name - monkeypatch.setattr(build_hooks.shutil, "which", fake_which) + monkeypatch.setattr(_build_shared.shutil, "which", fake_which) with pytest.raises(RuntimeError, match="clang and lld"): - build_hooks._check_toolchain_available("llvm") + _build_shared._check_toolchain_available("llvm") @pytest.mark.agent_authored(model="glm-5.2") def test_llvm_present_passes(self, monkeypatch): - monkeypatch.setattr(build_hooks.shutil, "which", lambda name: "/bin/" + name) - build_hooks._check_toolchain_available("llvm") + monkeypatch.setattr(_build_shared.shutil, "which", lambda name: "/bin/" + name) + _build_shared._check_toolchain_available("llvm") # --------------------------------------------------------------------------- diff --git a/ruff.toml b/ruff.toml index 210f852cd3e..db83fba5992 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 line-length = 120 @@ -73,7 +73,7 @@ ignore = [ exclude = ["**/_version.py"] [lint.isort] -known-first-party = ["cuda"] +known-first-party = ["cuda", "_build_shared"] [lint.flake8-quotes] inline-quotes = "double" diff --git a/toolshed/check_build_hooks_sync.py b/toolshed/check_build_hooks_sync.py index e1bfcd09183..a8aacf2dbd6 100644 --- a/toolshed/check_build_hooks_sync.py +++ b/toolshed/check_build_hooks_sync.py @@ -5,10 +5,9 @@ The block delimited by '# --- begin shared build helpers' and '# --- end shared build helpers ---' is duplicated verbatim between -cuda_bindings/build_hooks.py and cuda_core/build_hooks.py (PEP 517 build -isolation forbids a shared import). It contains the toolchain helpers and -the Cython cache helpers. Run as a pre-commit hook so drift is caught at -commit time. +cuda_bindings/build_hooks.py and cuda_core/build_hooks.py. It contains the +Cython cache helpers; the toolchain helpers already live in the shared +_build_shared.py. Run as a pre-commit hook so drift is caught at commit time. """ from __future__ import annotations From d3940dfa25084264b8a7a7113a7637009908a5ff Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 08:59:39 -0700 Subject: [PATCH 5/9] build: move Cython cache helpers and rebuild stamps into _build_shared.py The Cython cache helpers were duplicated verbatim in both build_hooks.py files, kept identical by a pre-commit sync check. The stamp mechanics (_BUILD_DIR, _abi_stamp_path, the force_build_ext flag, check/record of a stamped key) were near copies. Move them into _build_shared.py and delete toolshed/check_build_hooks_sync.py with its hook. Each build_hooks.py keeps only what is per package: its stamp file and key (toolchain for cuda-bindings; CUDA major, toolchain, debug and coverage for cuda-core). setup.py is unchanged: build_hooks re-exports force_build_ext through a module-level __getattr__. The Cython test builders now load _build_shared.py directly instead of build_hooks.py. The tests patch force_build_ext and sysconfig on _build_shared. --- .pre-commit-config.yaml | 7 - cuda_bindings/AGENTS.md | 7 +- cuda_bindings/_build_shared.py | 194 +++++++++++++++++ cuda_bindings/build_hooks.py | 191 ++--------------- cuda_bindings/tests/cython/build_tests.py | 32 ++- cuda_bindings/tests/test_build_hooks.py | 8 +- cuda_core/AGENTS.md | 6 +- cuda_core/build_hooks.py | 196 ++---------------- cuda_core/tests/cython/build_tests.py | 32 ++- cuda_core/tests/test_build_hooks.py | 10 +- .../cuda_python_test_helpers/cython_cache.py | 8 +- ruff.toml | 1 + toolshed/check_build_hooks_sync.py | 49 ----- 13 files changed, 283 insertions(+), 458 deletions(-) delete mode 100644 toolshed/check_build_hooks_sync.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebad9fb9097..640a4d4ff66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,13 +81,6 @@ repos: language: python files: '^cuda_core/tests/.*\.py$' - - id: check-build-hooks-sync - name: Check shared build_hooks.py blocks are in sync - entry: python ./toolshed/check_build_hooks_sync.py - language: python - files: '^(cuda_bindings|cuda_core)/build_hooks\.py$' - pass_filenames: false - - id: no-markdown-in-docs-source name: Prevent markdown files in docs/source directories entry: bash -c diff --git a/cuda_bindings/AGENTS.md b/cuda_bindings/AGENTS.md index 0f30fbb7154..e9e900c2185 100644 --- a/cuda_bindings/AGENTS.md +++ b/cuda_bindings/AGENTS.md @@ -19,9 +19,10 @@ subpackage in the `cuda-python` monorepo. - **Platform internals**: `cuda/bindings/_internal/` contains platform-specific implementation files and support code. - **Build backend**: `build_hooks.py` drives extension configuration and - Cythonization. Logic shared with `cuda_core` (toolchain selection and the - compiler flag set) lives in `_build_shared.py`; `cuda_core/_build_shared.py` - is a symlink to this file, so an edit here changes both packages. + Cythonization. Logic shared with `cuda_core` (toolchain selection, the + compiler flag set, the Cython cache helpers and the rebuild stamps) lives + in `_build_shared.py`; `cuda_core/_build_shared.py` is a symlink to this + file, so an edit here changes both packages. ## Generated-source workflow diff --git a/cuda_bindings/_build_shared.py b/cuda_bindings/_build_shared.py index 2d0440f65a3..9ec331204ee 100644 --- a/cuda_bindings/_build_shared.py +++ b/cuda_bindings/_build_shared.py @@ -17,15 +17,24 @@ ``resolve_toolchain``: the C++ standard, warnings-as-errors, and an optional ``tweak`` hook for flags a single package needs. +Besides the toolchain, this module owns the machinery that both backends use +around cythonize and build_ext: the opt-in Cython generated-source cache and +the build stamps that force a rebuild when the build configuration changed. + Note: There is no support guarantee for environment variables like CUDA_PYTHON_TOOLCHAIN. They may be removed or changed in the future. """ +import contextlib +import hashlib import os import shlex import shutil import sys import sysconfig +import uuid +from pathlib import Path +from warnings import warn # ----------------------------------------------------------------------- # Toolchain selection @@ -240,3 +249,188 @@ def resolve_toolchain(*, cxx_std, debug=False, compile_for_coverage=False, warni _apply_toolchain_env(cc, cxx, explicit) return name, cc, cxx, extra_compile_args, extra_link_args + + +# ----------------------------------------------------------------------- +# Cython cache helpers + + +# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === +# Workaround for Cython issue #7532: Cython's native cache fingerprint omits +# `compiler_directives`, so builds with different directives (e.g. linetrace +# for coverage) could reuse stale generated C/C++ output. This helper +# namespaces the Cython cache by package and a digest of output-affecting +# build configuration so distinct configurations get distinct caches. +# +# Removal: once cython/cython#7532 is resolved in a released Cython version +# and cuda-python's minimum Cython version includes the fix, this helper +# and its workaround-specific tests can be deleted; cythonize() can then be +# called with `cache=` (or `cache=True`) without per-config namespacing. +# See https://github.com/cython/cython/issues/7532 +def _cython_cache_path( + package, + *, + compiler_directives=None, + compile_time_env=None, + language_level=None, + cplus=None, + debug=False, + cuda_major=None, +): + """Return a per-configuration Cython cache directory, or None to disable caching. + + Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() + is called without ``cache=`` and existing workflows are unchanged. + """ + cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") + if not cache_root: + return None + if sys.platform == "win32": + warn( + "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " + "is not supported on Windows; caching will be disabled.", + stacklevel=2, + ) + return None + + h = hashlib.sha256() + h.update(package.encode("utf-8")) + # The Python version running cythonize affects generated C code + # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). + h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) + + def _update(name, value): + h.update(name.encode("utf-8")) + h.update(repr(value).encode("utf-8")) + + # compiler_directives are not in Cython's native fingerprint (#7532). + if compiler_directives: + for key in sorted(compiler_directives): + _update(f"directive:{key}", compiler_directives[key]) + # compile_time_env, language_level, and cplus are already in Cython's + # fingerprint, but we include them so the namespace stays correct even + # if Cython's fingerprint logic changes. + if compile_time_env: + for key in sorted(compile_time_env): + _update(f"compile_time_env:{key}", compile_time_env[key]) + if language_level is not None: + _update("language_level", language_level) + if cplus is not None: + _update("cplus", cplus) + # debug toggles gdb_debug in cythonize(), which affects generated code. + _update("debug", debug) + if cuda_major is not None: + _update("cuda_major", cuda_major) + + return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") + + +@contextlib.contextmanager +def _stable_cython_alias(target: Path, alias: Path): + """Atomically create a stable directory symlink alias for a Cython include tree. + + Cython's cache fingerprint includes the absolute path of each resolved + .pxd dependency (via ``file_hash()``). PEP 517 build environments install + dependencies under randomized temporary prefixes, making those paths + unstable across runs. This context manager creates a fixed, worktree- + relative symlink so Cython sees a stable lexical path. + + The symlink is created in the *package directory* (the directory containing + this file, which is the package's own copy through the symlink), not in + the cwd, to keep aliases package-local and avoid cross-package races. + + alias must not already exist as a real file or directory; if it is a + symlink (including a dangling one) it is atomically replaced. + + On exit the alias is removed only if it still points at ``target`` (a + racing replacement will not be deleted). + + POSIX only: directory symlinks require no elevated privileges on Linux. + """ + # Resolve the *parent* directory (must exist), then append the name. + # We deliberately do not follow a symlink that may already sit at alias. + if not alias.is_absolute(): + alias = Path(__file__).parent / alias + alias = alias.parent.resolve() / alias.name + target = target.resolve() + + if alias.exists() and not alias.is_symlink(): + raise RuntimeError( + f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." + ) + + tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") + try: + os.symlink(target, tmp_alias, target_is_directory=True) + try: + os.replace(tmp_alias, alias) + except BaseException: + tmp_alias.unlink(missing_ok=True) + raise + rel = os.path.relpath(alias, start=Path.cwd()) + yield rel + finally: + tmp_alias.unlink(missing_ok=True) + # Only remove the alias we created; leave it alone if something else + # has already replaced it (readlink will differ). + try: + if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: + alias.unlink() + except OSError: + pass + + +# ----------------------------------------------------------------------- +# Build stamps +# +# Setuptools' freshness check covers neither the extension flags nor the +# build configuration, so a stale .so built under another configuration looks +# perfectly fresh. Each backend stamps the key of its last completed build +# (the toolchain for cuda-bindings; CUDA major, toolchain and debug/coverage +# for cuda-core). check_build_key() sets ``force_build_ext`` when the key +# changed, and setup.py hands that to build_ext. + +# Where per-configuration build artifacts live. Anchored to this file rather +# than the cwd, since a project can be built from anywhere. +_BUILD_DIR = Path(__file__).parent / "build" + +# Set by check_build_key(). Read it as ``build_hooks.force_build_ext``: both +# build_hooks modules re-export it. +force_build_ext = False + + +def _abi_stamp_path(stem): + """Return a stamp path scoped to this interpreter's extension ABI.""" + extension_suffix = sysconfig.get_config_var("EXT_SUFFIX") + if not extension_suffix: + raise RuntimeError("Python's EXT_SUFFIX build configuration is unavailable") + return _BUILD_DIR / f"{stem}{extension_suffix}" + + +def check_build_key(stamp, key, description): + """Set force_build_ext when ``key`` differs from the one stamped at ``stamp``. + + ``description`` names the key in the message that explains the rebuild. + """ + global force_build_ext + + try: + previous = stamp.read_text(encoding="utf-8").strip() + except FileNotFoundError: + previous = None + + # A missing stamp means the last build's key is unknown, so force too. + # On a first build that costs nothing: there are no artifacts to reuse. + if previous != key: + print(f"{description} of last build: {previous} (building {key}); forcing a full rebuild") + force_build_ext = True + + +def record_build_key(stamp, key): + """Stamp ``key`` at ``stamp``, once the build it describes has completed. + + A build that failed partway through must not claim outputs it never + produced, so callers record only after the build succeeded. + """ + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text(key + "\n", encoding="utf-8") diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index 08a1a2344c3..af56bec3aab 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -11,21 +11,29 @@ import contextlib import functools import glob -import hashlib import os import re import shutil import sys import sysconfig import tempfile -import uuid from pathlib import Path from warnings import warn from setuptools import build_meta as _build_meta from setuptools.extension import Extension -from _build_shared import _check_toolchain_available, _resolve_toolchain_name, resolve_toolchain +import _build_shared +from _build_shared import ( + _abi_stamp_path, + _check_toolchain_available, + _cython_cache_path, + _resolve_toolchain_name, + _stable_cython_alias, + check_build_key, + record_build_key, + resolve_toolchain, +) # Metadata hooks delegate directly to setuptools -- no CUDA needed. prepare_metadata_for_build_editable = _build_meta.prepare_metadata_for_build_editable @@ -35,6 +43,15 @@ get_requires_for_build_wheel = _build_meta.get_requires_for_build_wheel get_requires_for_build_editable = _build_meta.get_requires_for_build_editable + +def __getattr__(name): + # setup.py reads ``build_hooks.force_build_ext``; the flag itself lives in + # _build_shared, where check_build_key() sets it. + if name == "force_build_ext": + return _build_shared.force_build_ext + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + # Note: There is no support guarantee for environment variables like # CUDA_PYTHON_TOOLCHAIN, CUDA_PYTHON_CYTHON_CACHE_DIR, etc. They may be # removed or changed in the future. @@ -165,145 +182,6 @@ def _check_cuda_headers(cuda_path: str) -> None: ) -# ----------------------------------------------------------------------- -# Cython cache helpers -# -# The toolchain helpers and flag assembly live in _build_shared.py (the -# single source of truth; cuda_core/_build_shared.py is a symlink to it). -# The helper block below is still duplicated verbatim in -# cuda_{bindings,core}/build_hooks.py (keep it in sync; enforced by -# toolshed/check_build_hooks_sync.py). - - -# --- begin shared build helpers (keep in sync) --- -# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === -# Workaround for Cython issue #7532: Cython's native cache fingerprint omits -# `compiler_directives`, so builds with different directives (e.g. linetrace -# for coverage) could reuse stale generated C/C++ output. This helper -# namespaces the Cython cache by package and a digest of output-affecting -# build configuration so distinct configurations get distinct caches. -# -# Removal: once cython/cython#7532 is resolved in a released Cython version -# and cuda-python's minimum Cython version includes the fix, this helper -# and its workaround-specific tests can be deleted; cythonize() can then be -# called with `cache=` (or `cache=True`) without per-config namespacing. -# See https://github.com/cython/cython/issues/7532 -def _cython_cache_path( - package, - *, - compiler_directives=None, - compile_time_env=None, - language_level=None, - cplus=None, - debug=False, - cuda_major=None, -): - """Return a per-configuration Cython cache directory, or None to disable caching. - - Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() - is called without ``cache=`` and existing workflows are unchanged. - """ - cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") - if not cache_root: - return None - if sys.platform == "win32": - warn( - "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " - "is not supported on Windows; caching will be disabled.", - stacklevel=2, - ) - return None - - h = hashlib.sha256() - h.update(package.encode("utf-8")) - # The Python version running cythonize affects generated C code - # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). - h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) - - def _update(name, value): - h.update(name.encode("utf-8")) - h.update(repr(value).encode("utf-8")) - - # compiler_directives are not in Cython's native fingerprint (#7532). - if compiler_directives: - for key in sorted(compiler_directives): - _update(f"directive:{key}", compiler_directives[key]) - # compile_time_env, language_level, and cplus are already in Cython's - # fingerprint, but we include them so the namespace stays correct even - # if Cython's fingerprint logic changes. - if compile_time_env: - for key in sorted(compile_time_env): - _update(f"compile_time_env:{key}", compile_time_env[key]) - if language_level is not None: - _update("language_level", language_level) - if cplus is not None: - _update("cplus", cplus) - # debug toggles gdb_debug in cythonize(), which affects generated code. - _update("debug", debug) - if cuda_major is not None: - _update("cuda_major", cuda_major) - - return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") - - -@contextlib.contextmanager -def _stable_cython_alias(target: Path, alias: Path): - """Atomically create a stable directory symlink alias for a Cython include tree. - - Cython's cache fingerprint includes the absolute path of each resolved - .pxd dependency (via ``file_hash()``). PEP 517 build environments install - dependencies under randomized temporary prefixes, making those paths - unstable across runs. This context manager creates a fixed, worktree- - relative symlink so Cython sees a stable lexical path. - - The symlink is created in the *package directory* (the directory containing - this build_hooks.py), not in the cwd, to keep aliases package-local and - avoid cross-package races. - - alias must not already exist as a real file or directory; if it is a - symlink (including a dangling one) it is atomically replaced. - - On exit the alias is removed only if it still points at ``target`` (a - racing replacement will not be deleted). - - POSIX only: directory symlinks require no elevated privileges on Linux. - """ - # Resolve the *parent* directory (must exist), then append the name. - # We deliberately do not follow a symlink that may already sit at alias. - if not alias.is_absolute(): - alias = Path(__file__).parent / alias - alias = alias.parent.resolve() / alias.name - target = target.resolve() - - if alias.exists() and not alias.is_symlink(): - raise RuntimeError( - f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." - ) - - tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") - try: - os.symlink(target, tmp_alias, target_is_directory=True) - try: - os.replace(tmp_alias, alias) - except BaseException: - tmp_alias.unlink(missing_ok=True) - raise - rel = os.path.relpath(alias, start=Path.cwd()) - yield rel - finally: - tmp_alias.unlink(missing_ok=True) - # Only remove the alias we created; leave it alone if something else - # has already replaced it (readlink will differ). - try: - if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: - alias.unlink() - except OSError: - pass - - -# --- end shared build helpers --- - - def _tweak_flags(name, extra_compile_args, extra_link_args): """cuda-bindings flags that do not belong in the shared set.""" if name != "msvc": @@ -333,24 +211,11 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False): # ----------------------------------------------------------------------- # Toolchain stamp -_BUILD_DIR = Path(__file__).parent / "build" - - -def _abi_stamp_path(stem): - """Return a stamp path scoped to this interpreter's extension ABI.""" - extension_suffix = sysconfig.get_config_var("EXT_SUFFIX") - if not extension_suffix: - raise RuntimeError("Python's EXT_SUFFIX build configuration is unavailable") - return _BUILD_DIR / f"{stem}{extension_suffix}" - - # Records the toolchain of the last completed build for this extension ABI, # so setup.py can force build_ext when it changes. Written by # record_build_toolchain(). _BUILD_TOOLCHAIN_STAMP = _abi_stamp_path(".build-toolchain") -force_build_ext = False - def _check_build_toolchain(toolchain): """Set force_build_ext when the toolchain changed since the last build. @@ -358,18 +223,7 @@ def _check_build_toolchain(toolchain): Setuptools' freshness check does not include the extension flags, so a stale .so compiled by a previous toolchain would otherwise be packaged. """ - global force_build_ext - - try: - previous = _BUILD_TOOLCHAIN_STAMP.read_text(encoding="utf-8").strip() - except FileNotFoundError: - previous = None - - # A missing stamp means the last build's toolchain is unknown, so force too. - # On a first build that costs nothing: there are no artifacts to reuse. - if previous != toolchain: - print(f"Toolchain of last build: {previous} (building {toolchain}); forcing a full rebuild") - force_build_ext = True + check_build_key(_BUILD_TOOLCHAIN_STAMP, toolchain, "Toolchain") def record_build_toolchain() -> None: @@ -380,8 +234,7 @@ def record_build_toolchain() -> None: toolchain name from the environment rather than caching it in a global. """ name, *_ = _resolve_toolchain_name() - _BUILD_TOOLCHAIN_STAMP.parent.mkdir(parents=True, exist_ok=True) - _BUILD_TOOLCHAIN_STAMP.write_text(name + "\n", encoding="utf-8") + record_build_key(_BUILD_TOOLCHAIN_STAMP, name) # ----------------------------------------------------------------------- diff --git a/cuda_bindings/tests/cython/build_tests.py b/cuda_bindings/tests/cython/build_tests.py index 9943f74f66d..9ddfadfbad8 100644 --- a/cuda_bindings/tests/cython/build_tests.py +++ b/cuda_bindings/tests/cython/build_tests.py @@ -10,7 +10,7 @@ it via `include_path=` so cythonize finds the .pxd tree on every platform. When CUDA_PYTHON_CYTHON_CACHE_DIR is set, cythonize uses the same cache -namespacing and include-path aliasing as cuda_bindings/build_hooks.py. +namespacing and include-path aliasing as the package build (``_build_shared.py``). """ from __future__ import annotations @@ -29,26 +29,18 @@ _COMPILER_DIRECTIVES = {"freethreading_compatible": True} -def _load_module(name, path, *, register=False): - # PEP 517 backend files, not installed modules. Load by path so we do not - # put the package directory on sys.path (that would shadow the installed - # package). With ``register`` the module is also entered into sys.modules, - # which is how build_hooks.py's ``from _build_shared import ...`` finds it. - spec = importlib.util.spec_from_file_location(name, path) +def _load_build_shared(): + # A PEP 517 backend file, not an installed module. Load it by path so we do + # not put the package directory on sys.path (that would shadow the + # installed package). It needs only the standard library. + path = Path(__file__).resolve().parents[2] / "_build_shared.py" + spec = importlib.util.spec_from_file_location("cuda_bindings_build_shared", path) module = importlib.util.module_from_spec(spec) - if register: - sys.modules[name] = module spec.loader.exec_module(module) return module -def _load_build_hooks(modname): - package_root = Path(__file__).resolve().parents[2] - _load_module("_build_shared", package_root / "_build_shared.py", register=True) - return _load_module(modname, package_root / "build_hooks.py") - - -build_hooks = _load_build_hooks("cuda_bindings_build_hooks") +build_shared = _load_build_shared() def _bindings_source_root() -> Path: @@ -62,7 +54,7 @@ def _bindings_source_root() -> Path: def _cythonize_tests(pyx_files): - cache_path = build_hooks._cython_cache_path( + cache_path = build_shared._cython_cache_path( "cuda-bindings-cython-tests", compiler_directives=_COMPILER_DIRECTIVES, language_level=3, @@ -83,11 +75,11 @@ def _cythonize_tests(pyx_files): # Distinct alias names so a concurrent package build's .cython-stdlib / # .cython-bindings symlinks are not replaced. Relative aliases resolve - # next to build_hooks.py (package root). + # next to _build_shared.py (package root). stdlib_target = Path(Cython.__file__).parent / "Includes" with ( - build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, - build_hooks._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, + build_shared._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, + build_shared._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, ): return cythonize( pyx_files, diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index 4f522c5b8f6..29078f2ac46 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -71,7 +71,7 @@ def _isolate_toolchain_env(): def _fake_sysconfig(monkeypatch, **values): """Pin sysconfig.get_config_var so linker-command assertions are exact.""" - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda name: values.get(name)) + monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) class TestResolveToolchain: @@ -392,7 +392,7 @@ def stamp(tmp_path, monkeypatch): """Redirect the toolchain stamp to a scratch path.""" scratch = tmp_path / "build" / ".build-toolchain" monkeypatch.setattr(build_hooks, "_BUILD_TOOLCHAIN_STAMP", scratch) - monkeypatch.setattr(build_hooks, "force_build_ext", False) + monkeypatch.setattr(_build_shared, "force_build_ext", False) monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) return scratch @@ -407,9 +407,9 @@ class TestBuildToolchainStamp: @pytest.mark.agent_authored(model="grok-4.6") def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") + monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") python_310 = build_hooks._abi_stamp_path(".build-toolchain") - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") + monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") python_311 = build_hooks._abi_stamp_path(".build-toolchain") assert python_310 != python_311 diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index fac6d1b2a85..41f90708271 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -23,9 +23,9 @@ This file describes `cuda_core`, the high-level Pythonic CUDA subpackage in the one `_cpp/.cpp` or as a directory `_cpp//` whose sources all compile into the `_` extension (`_cpp/rt/` for `_rt`). - **Build backend**: `build_hooks.py` handles Cython extension setup and build - dependency wiring. Logic shared with `cuda_bindings` (toolchain selection and - the compiler flag set) lives in `_build_shared.py`, a symlink to - `cuda_bindings/_build_shared.py`; an edit through either path changes both + dependency wiring. Logic shared with `cuda_bindings` (toolchain selection, + the compiler flag set, the Cython cache helpers and the rebuild stamps) + lives in `_build_shared.py`, a symlink to `cuda_bindings/_build_shared.py`; an edit through either path changes both packages. ## Build and version coupling diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index 009825b51f6..824b0a90fe2 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -7,20 +7,15 @@ # - https://setuptools.pypa.io/en/latest/build_meta.html#dynamic-build-dependencies-and-other-build-meta-tweaks # Specifically, there are 5 APIs required to create a proper build backend, see below. -import contextlib import functools import glob -import hashlib import importlib.util import os import re import sys -import sysconfig import tempfile -import uuid import zipfile from pathlib import Path -from warnings import warn import Cython as _Cython from Cython.Build import cythonize @@ -28,13 +23,32 @@ from setuptools import Extension from setuptools import build_meta as _build_meta -from _build_shared import _check_toolchain_available, resolve_toolchain +import _build_shared +from _build_shared import ( + _BUILD_DIR, + _abi_stamp_path, + _check_toolchain_available, + _cython_cache_path, + _stable_cython_alias, + check_build_key, + record_build_key, + resolve_toolchain, +) prepare_metadata_for_build_editable = _build_meta.prepare_metadata_for_build_editable prepare_metadata_for_build_wheel = _build_meta.prepare_metadata_for_build_wheel build_sdist = _build_meta.build_sdist get_requires_for_build_sdist = _build_meta.get_requires_for_build_sdist + +def __getattr__(name): + # setup.py reads ``build_hooks.force_build_ext``; the flag itself lives in + # _build_shared, where check_build_key() sets it. + if name == "force_build_ext": + return _build_shared.force_build_ext + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + # Note: There is no support guarantee for environment variables like CUDA_PYTHON_COVERAGE, # CUDA_PYTHON_TOOLCHAIN, CUDA_PYTHON_CYTHON_CACHE_DIR, etc. They may be removed # or changed in the future. @@ -211,145 +225,6 @@ def _read_cuda_h_version(cuda_path: str) -> int: ) -# ----------------------------------------------------------------------- -# Cython cache helpers -# -# The toolchain helpers and flag assembly live in _build_shared.py (the -# single source of truth; cuda_core/_build_shared.py is a symlink to it). -# The helper block below is still duplicated verbatim in -# cuda_{bindings,core}/build_hooks.py (keep it in sync; enforced by -# toolshed/check_build_hooks_sync.py). - - -# --- begin shared build helpers (keep in sync) --- -# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === -# Workaround for Cython issue #7532: Cython's native cache fingerprint omits -# `compiler_directives`, so builds with different directives (e.g. linetrace -# for coverage) could reuse stale generated C/C++ output. This helper -# namespaces the Cython cache by package and a digest of output-affecting -# build configuration so distinct configurations get distinct caches. -# -# Removal: once cython/cython#7532 is resolved in a released Cython version -# and cuda-python's minimum Cython version includes the fix, this helper -# and its workaround-specific tests can be deleted; cythonize() can then be -# called with `cache=` (or `cache=True`) without per-config namespacing. -# See https://github.com/cython/cython/issues/7532 -def _cython_cache_path( - package, - *, - compiler_directives=None, - compile_time_env=None, - language_level=None, - cplus=None, - debug=False, - cuda_major=None, -): - """Return a per-configuration Cython cache directory, or None to disable caching. - - Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() - is called without ``cache=`` and existing workflows are unchanged. - """ - cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") - if not cache_root: - return None - if sys.platform == "win32": - warn( - "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " - "is not supported on Windows; caching will be disabled.", - stacklevel=2, - ) - return None - - h = hashlib.sha256() - h.update(package.encode("utf-8")) - # The Python version running cythonize affects generated C code - # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). - h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) - - def _update(name, value): - h.update(name.encode("utf-8")) - h.update(repr(value).encode("utf-8")) - - # compiler_directives are not in Cython's native fingerprint (#7532). - if compiler_directives: - for key in sorted(compiler_directives): - _update(f"directive:{key}", compiler_directives[key]) - # compile_time_env, language_level, and cplus are already in Cython's - # fingerprint, but we include them so the namespace stays correct even - # if Cython's fingerprint logic changes. - if compile_time_env: - for key in sorted(compile_time_env): - _update(f"compile_time_env:{key}", compile_time_env[key]) - if language_level is not None: - _update("language_level", language_level) - if cplus is not None: - _update("cplus", cplus) - # debug toggles gdb_debug in cythonize(), which affects generated code. - _update("debug", debug) - if cuda_major is not None: - _update("cuda_major", cuda_major) - - return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") - - -@contextlib.contextmanager -def _stable_cython_alias(target: Path, alias: Path): - """Atomically create a stable directory symlink alias for a Cython include tree. - - Cython's cache fingerprint includes the absolute path of each resolved - .pxd dependency (via ``file_hash()``). PEP 517 build environments install - dependencies under randomized temporary prefixes, making those paths - unstable across runs. This context manager creates a fixed, worktree- - relative symlink so Cython sees a stable lexical path. - - The symlink is created in the *package directory* (the directory containing - this build_hooks.py), not in the cwd, to keep aliases package-local and - avoid cross-package races. - - alias must not already exist as a real file or directory; if it is a - symlink (including a dangling one) it is atomically replaced. - - On exit the alias is removed only if it still points at ``target`` (a - racing replacement will not be deleted). - - POSIX only: directory symlinks require no elevated privileges on Linux. - """ - # Resolve the *parent* directory (must exist), then append the name. - # We deliberately do not follow a symlink that may already sit at alias. - if not alias.is_absolute(): - alias = Path(__file__).parent / alias - alias = alias.parent.resolve() / alias.name - target = target.resolve() - - if alias.exists() and not alias.is_symlink(): - raise RuntimeError( - f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." - ) - - tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") - try: - os.symlink(target, tmp_alias, target_is_directory=True) - try: - os.replace(tmp_alias, alias) - except BaseException: - tmp_alias.unlink(missing_ok=True) - raise - rel = os.path.relpath(alias, start=Path.cwd()) - yield rel - finally: - tmp_alias.unlink(missing_ok=True) - # Only remove the alias we created; leave it alone if something else - # has already replaced it (readlink will differ). - try: - if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: - alias.unlink() - except OSError: - pass - - -# --- end shared build helpers --- - - def _resolve_toolchain(debug=False, compile_for_coverage=False): """Resolve the C/C++ toolchain from CUDA_PYTHON_TOOLCHAIN (cuda.core flags). @@ -491,27 +366,12 @@ def _build_define_macros(cuda_major: str, bindings_cuda_version: int | None = No # used later by setup() _extensions = None -# Where per-configuration build artifacts live. Anchored to this file rather -# than the cwd, since a project can be built from anywhere. -_BUILD_DIR = Path(__file__).parent / "build" - - -def _abi_stamp_path(stem): - """Return a stamp path scoped to this interpreter's extension ABI.""" - extension_suffix = sysconfig.get_config_var("EXT_SUFFIX") - if not extension_suffix: - raise RuntimeError("Python's EXT_SUFFIX build configuration is unavailable") - return _BUILD_DIR / f"{stem}{extension_suffix}" - - # Records the build configuration (CUDA major, toolchain, debug/coverage) of # the last completed build for this extension ABI, so setup.py can force # build_ext when it changes. Written by record_build_config() after the # PEP 517 backend succeeds. _BUILD_CONFIG_STAMP = _abi_stamp_path(".build-config") -force_build_ext = False - def _build_config_key(cuda_major, toolchain, debug, coverage): """Return a stable string key for the build configuration.""" @@ -530,20 +390,9 @@ def _check_build_config(toolchain, debug, coverage): major, toolchain, debug/coverage) is therefore stamped and build_ext forced whenever it changes, so a stale .so is never packaged. """ - global force_build_ext - cuda_major = _determine_cuda_major_version() key = _build_config_key(cuda_major, toolchain, debug, coverage) - try: - previous = _BUILD_CONFIG_STAMP.read_text(encoding="utf-8").strip() - except FileNotFoundError: - previous = None - - # A missing stamp means the last build's config is unknown, so force too. - # On a first build that costs nothing: there are no artifacts to reuse. - if previous != key: - print(f"Build config of last build: {previous} (building {key}); forcing a full rebuild") - force_build_ext = True + check_build_key(_BUILD_CONFIG_STAMP, key, "Build config") return cuda_major, key @@ -555,8 +404,7 @@ def record_build_config(key) -> None: passing the key already checked rather than re-deriving from ambient state (setuptools' `build_ext.debug` is not `config_settings["debug"]`). """ - _BUILD_CONFIG_STAMP.parent.mkdir(parents=True, exist_ok=True) - _BUILD_CONFIG_STAMP.write_text(key + "\n", encoding="utf-8") + record_build_key(_BUILD_CONFIG_STAMP, key) def _relativize_extension_sources(extensions) -> None: diff --git a/cuda_core/tests/cython/build_tests.py b/cuda_core/tests/cython/build_tests.py index 453cf3c7915..39539e3100c 100644 --- a/cuda_core/tests/cython/build_tests.py +++ b/cuda_core/tests/cython/build_tests.py @@ -10,7 +10,7 @@ it via `include_path=` so cythonize finds the .pxd tree on every platform. When CUDA_PYTHON_CYTHON_CACHE_DIR is set, cythonize uses the same cache -namespacing and include-path aliasing as cuda_core/build_hooks.py. +namespacing and include-path aliasing as the package build (``_build_shared.py``). """ from __future__ import annotations @@ -29,26 +29,18 @@ _COMPILER_DIRECTIVES = {"freethreading_compatible": True} -def _load_module(name, path, *, register=False): - # PEP 517 backend files, not installed modules. Load by path so we do not - # put the package directory on sys.path (that would shadow the installed - # package). With ``register`` the module is also entered into sys.modules, - # which is how build_hooks.py's ``from _build_shared import ...`` finds it. - spec = importlib.util.spec_from_file_location(name, path) +def _load_build_shared(): + # A PEP 517 backend file, not an installed module. Load it by path so we do + # not put the package directory on sys.path (that would shadow the + # installed package). It needs only the standard library. + path = Path(__file__).resolve().parents[2] / "_build_shared.py" + spec = importlib.util.spec_from_file_location("cuda_core_build_shared", path) module = importlib.util.module_from_spec(spec) - if register: - sys.modules[name] = module spec.loader.exec_module(module) return module -def _load_build_hooks(modname): - package_root = Path(__file__).resolve().parents[2] - _load_module("_build_shared", package_root / "_build_shared.py", register=True) - return _load_module(modname, package_root / "build_hooks.py") - - -build_hooks = _load_build_hooks("cuda_core_build_hooks") +build_shared = _load_build_shared() def _bindings_source_root() -> Path: @@ -62,7 +54,7 @@ def _bindings_source_root() -> Path: def _cythonize_tests(pyx_files): - cache_path = build_hooks._cython_cache_path( + cache_path = build_shared._cython_cache_path( "cuda-core-cython-tests", compiler_directives=_COMPILER_DIRECTIVES, language_level=3, @@ -82,11 +74,11 @@ def _cythonize_tests(pyx_files): # Distinct alias names so a concurrent package build's .cython-stdlib / # .cython-bindings symlinks are not replaced. Relative aliases resolve - # next to build_hooks.py (package root). + # next to _build_shared.py (package root). stdlib_target = Path(Cython.__file__).parent / "Includes" with ( - build_hooks._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, - build_hooks._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, + build_shared._stable_cython_alias(stdlib_target, Path(".cython-stdlib-tests")) as rel_stdlib, + build_shared._stable_cython_alias(_bindings_source_root(), Path(".cython-bindings-tests")) as rel_bindings, ): return cythonize( pyx_files, diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 9883da3c26b..b914896c35f 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -81,7 +81,7 @@ def _isolate_toolchain_env(): def _fake_sysconfig(monkeypatch, **values): """Pin sysconfig.get_config_var so linker-command assertions are exact.""" - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda name: values.get(name)) + monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) @pytest.mark.agent_authored(model="gpt-5.6") @@ -211,7 +211,7 @@ def stamp(tmp_path, monkeypatch): """ scratch = tmp_path / "build" / ".build-config" monkeypatch.setattr(build_hooks, "_BUILD_CONFIG_STAMP", scratch) - monkeypatch.setattr(build_hooks, "force_build_ext", False) + monkeypatch.setattr(_build_shared, "force_build_ext", False) build_hooks._get_cuda_path.cache_clear() build_hooks._determine_cuda_major_version.cache_clear() get_cuda_path_or_home.cache_clear() @@ -231,9 +231,9 @@ class TestBuildConfigStamp: @pytest.mark.agent_authored(model="grok-4.6") def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") + monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") python_310 = build_hooks._abi_stamp_path(".build-config") - monkeypatch.setattr(build_hooks.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") + monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") python_311 = build_hooks._abi_stamp_path(".build-config") assert python_310 != python_311 @@ -494,7 +494,7 @@ def _finalized_build_ext(force_flag, monkeypatch): setup_py = _load_setup_py(monkeypatch) assert setup_py.build_hooks is build_hooks - monkeypatch.setattr(build_hooks, "force_build_ext", force_flag) + monkeypatch.setattr(_build_shared, "force_build_ext", force_flag) cmd = setup_py.build_ext(Distribution({"name": "cuda-core", "version": "0"})) cmd.finalize_options() diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py b/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py index e22687055d7..997f5a75dc6 100644 --- a/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py +++ b/cuda_python_test_helpers/cuda_python_test_helpers/cython_cache.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared tests for the Cython cache helpers in build_hooks.py. +"""Shared tests for the Cython cache helpers in _build_shared.py. Provides: @@ -14,9 +14,9 @@ disable-path tests (warn + return None when the cache dir is set). These are used by ``cuda_bindings/tests/test_build_hooks.py`` -and ``cuda_core/tests/test_build_hooks.py``. Drift between the two vendored -helper copies is enforced by ``toolshed/check_build_hooks_sync.py``, not by a -runtime test. +and ``cuda_core/tests/test_build_hooks.py``. The helpers themselves live in +``_build_shared.py``, which ``cuda_core`` shares with ``cuda_bindings`` through +a symlink. Cython is imported inside the functions that need it so this module does not force a Cython dependency on the ``cuda-python-test-helpers`` package. diff --git a/ruff.toml b/ruff.toml index db83fba5992..7523b3450cf 100644 --- a/ruff.toml +++ b/ruff.toml @@ -143,5 +143,6 @@ inline-quotes = "double" "toolshed/**" = ["T201"] "ci/**" = ["T201"] "**/build_hooks.py" = ["T201"] +"**/_build_shared.py" = ["T201"] "**/docs/**/conf.py" = ["T201", "ARG001"] "cuda_pathfinder/**/dynamic_lib_subprocess.py" = ["T201"] diff --git a/toolshed/check_build_hooks_sync.py b/toolshed/check_build_hooks_sync.py deleted file mode 100644 index a8aacf2dbd6..00000000000 --- a/toolshed/check_build_hooks_sync.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Check that the shared build-helpers block is byte-identical in both build_hooks.py files. - -The block delimited by '# --- begin shared build helpers' and -'# --- end shared build helpers ---' is duplicated verbatim between -cuda_bindings/build_hooks.py and cuda_core/build_hooks.py. It contains the -Cython cache helpers; the toolchain helpers already live in the shared -_build_shared.py. Run as a pre-commit hook so drift is caught at commit time. -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -_MARKER_START = "# --- begin shared build helpers" -_MARKER_END = "# --- end shared build helpers ---" - -ROOT = Path(__file__).resolve().parents[1] -_BINDINGS = ROOT / "cuda_bindings" / "build_hooks.py" -_CORE = ROOT / "cuda_core" / "build_hooks.py" - - -def _shared_block(path: Path) -> str: - text = path.read_text(encoding="utf-8") - try: - start = text.index(_MARKER_START) - end = text.index(_MARKER_END) + len(_MARKER_END) - except ValueError as exc: - sys.exit(f"ERROR: sync marker not found in {path}: {exc}") - return text[start:end] - - -def main() -> None: - bindings_block = _shared_block(_BINDINGS) - core_block = _shared_block(_CORE) - if bindings_block != core_block: - sys.exit( - "ERROR: shared build helpers are out of sync between\n" - f" {_BINDINGS}\n" - f" {_CORE}\n" - "Edit both files to match and commit again." - ) - - -if __name__ == "__main__": - main() From e7c0f096c903aec895e51fab44a2801a7df586e0 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 09:21:28 -0700 Subject: [PATCH 6/9] test: share _build_shared tests through mixins and prune wheel-covered ones The toolchain, linker-command and stamp tests were duplicated verbatim in cuda_bindings/tests/test_build_hooks.py and cuda_core/tests/test_build_hooks.py, although they exercise code that now lives in the one _build_shared.py. Move them into mixins in cuda_python_test_helpers/build_shared.py, which each package mixes in against the _build_shared module it loaded. Each package keeps only its own choices: bindings' c++14 and -Wno-deprecated-declarations, core's c++17 and CUDA_PYTHON_WERROR wiring, and the key each one stamps. Prune what a wheel build already proves: the llvm preflight being a no-op for the default toolchain, and the default toolchain's name and compiler binaries. Add tests for behavior that had none: the Werror flags per platform, the check_build_key/record_build_key protocol, and that build_hooks re-exports force_build_ext from _build_shared without holding a copy. Patching the flag on build_hooks instead would leave a plain attribute there on teardown that shadows the re-export for later tests, so the tests patch _build_shared. --- cuda_bindings/tests/test_build_hooks.py | 405 +++----------- cuda_core/tests/test_build_hooks.py | 384 +++----------- .../cuda_python_test_helpers/build_shared.py | 499 ++++++++++++++++++ 3 files changed, 636 insertions(+), 652 deletions(-) create mode 100644 cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index 29078f2ac46..182f4d390c3 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -16,9 +16,7 @@ import importlib.util import os -import shlex import sys -import sysconfig from pathlib import Path # build_hooks.py imports Cython and setuptools at the top level; both are @@ -27,8 +25,6 @@ import Cython # noqa: F401 import pytest import setuptools # noqa: F401 -from setuptools._distutils.ccompiler import new_compiler -from setuptools._distutils.sysconfig import customize_compiler def _load_module(name, path, *, register=False): @@ -74,322 +70,9 @@ def _fake_sysconfig(monkeypatch, **values): monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) -class TestResolveToolchain: - """_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN. - - The default toolchain (gnu on Linux, msvc on Windows) must reproduce the - previous build behavior exactly and must not touch CC/CXX/LDCXXSHARED, so an - externally-set compiler (e.g. the sccache wrapper in CI) survives. - """ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_does_not_touch_env(self, monkeypatch): - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - monkeypatch.delenv("LDCXXSHARED", raising=False) - name, cc, cxx, _cargs, _largs = build_hooks._resolve_toolchain() - if sys.platform == "win32": - assert name == "msvc" - assert cc is None and cxx is None - else: - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert "CC" not in os.environ and "CXX" not in os.environ - assert "LDCXXSHARED" not in os.environ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_preserves_existing_cc(self, monkeypatch): - # An externally-set CC (e.g. sccache) must survive the default toolchain. - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache cc" - assert os.environ["CXX"] == "sccache c++" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_case_insensitive(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "LLVM") - name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert name == "llvm" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_invalid_value_raises(self, monkeypatch): - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "icc") - with pytest.raises(RuntimeError, match="not supported"): - build_hooks._resolve_toolchain() - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "llvm" - assert (cc, cxx) == ("clang", "clang++") - assert os.environ["CC"] == "clang" - assert os.environ["CXX"] == "clang++" - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - assert "-fuse-ld=lld" in largs - # clang rejects the gcc-only flags that gnu uses; they must be absent. - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="glm-5.2") - def test_gnu_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") - _fake_sysconfig(monkeypatch, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert os.environ["CC"] == "gcc" - assert os.environ["CXX"] == "g++" - assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="grok-4.6") - def test_llvm_keeps_sccache_prefix(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache clang" - assert os.environ["CXX"] == "sccache clang++" - assert "LDSHARED" not in os.environ - # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_linux_opt_flag_set(self, monkeypatch): - """Linux opt build: -std=c++14, -O2, -Wno-deprecated-declarations; no -O3 or gnu-only flags.""" - if sys.platform == "win32": - pytest.skip("Linux flags only") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain(debug=False) - assert "-std=c++14" in cargs - assert "-Wno-deprecated-declarations" in cargs - assert "-g0" in cargs - assert "-O2" in cargs - assert "-O3" not in cargs - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - assert "-Wl,--strip-all" in largs - - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_msvc_opt_flag_set(self, monkeypatch): - """MSVC opt build emits /std:c++14 and /O2.""" - if sys.platform != "win32": - pytest.skip("MSVC flags only on Windows") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) - assert "/std:c++14" in cargs - assert "/O2" in cargs - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDSHARED="gcc -shared -Wl,-z,relro") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" - _fake_sysconfig(monkeypatch) - os.environ.pop("LDCXXSHARED") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared" - - -class TestWithSccache: - """_with_sccache: keep a leading sccache token, swap the compiler.""" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_sccache_and_swaps_compiler(self): - assert _build_shared._with_sccache("sccache cc", "clang") == "sccache clang" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_absolute_sccache_path(self): - assert ( - _build_shared._with_sccache("/host/usr/local/bin/sccache cc", "clang") - == "/host/usr/local/bin/sccache clang" - ) - - @pytest.mark.agent_authored(model="grok-4.6") - def test_bare_or_unrelated_cc_returns_compiler(self): - assert _build_shared._with_sccache("", "clang") == "clang" - assert _build_shared._with_sccache("gcc", "clang") == "clang" - assert _build_shared._with_sccache("ccache gcc", "clang") == "clang" - - -class TestWithCompiler: - """_with_compiler: replace the compiler executable, keep following flags.""" - - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): - assert ( - _build_shared._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") - == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" - ) - - @pytest.mark.agent_authored(model="grok-4.6") - def test_compiler_only_command_returns_compiler(self): - assert _build_shared._with_compiler("g++", "clang++") == "clang++" - assert _build_shared._with_compiler("", "clang++") == "clang++" - assert _build_shared._with_compiler(None, "clang++") == "clang++" - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_quoted_arguments_intact(self): - result = _build_shared._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") - assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_drops_launcher_before_compiler(self): - # setuptools takes the launcher from CXX; keeping a second copy here would - # leave a stray compiler argument on the link line. - assert _build_shared._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_env_prefix(self): - assert ( - _build_shared._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") - == "env LIBRARY_PATH=/custom/lib clang++ -shared" - ) - # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env - assert ( - _build_shared._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") - == "env --unset=LD_LIBRARY_PATH clang++ -shared" - ) - - -class TestDistutilsLinkerIntegration: - """The env set by _resolve_toolchain, as setuptools' distutils consumes it. - - The tests above check os.environ; this checks the linker commands distutils - derives from it, which is what actually reaches the C++ link step. - """ - - @staticmethod - def _customized_compiler(): - compiler = new_compiler() - customize_compiler(compiler) - if not hasattr(compiler, "linker_so_cxx"): - pytest.skip("this setuptools' distutils has no linker_so_cxx") - return compiler - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") - if not sysconfig_ld: - pytest.skip("this Python has no LDCXXSHARED") - # Everything from the first flag on, including operands such as ``-B /path``. - tokens = shlex.split(sysconfig_ld) - first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) - expected_tail = tokens[first_flag:] - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[0] == "clang++" - assert linker[1 : 1 + len(expected_tail)] == expected_tail - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:6] == [ - "clang++", - "-pthread", - "-B", - "/path/to/python_compiler_compat", - "-shared", - "-Wl,-rpath,/lib", - ] - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] - assert "g++" not in linker - - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - build_hooks._resolve_toolchain() - compiler = self._customized_compiler() - assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] - linker = compiler.linker_so_cxx - assert linker[0] == "clang++" - assert linker.count("clang++") == 1 - - -class TestCheckToolchainAvailable: - """_check_toolchain_available: fast, helpful failure when a tool is missing.""" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_is_noop(self): - _build_shared._check_toolchain_available("gnu") - _build_shared._check_toolchain_available("msvc") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): - def fake_which(name): - return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name - - monkeypatch.setattr(_build_shared.shutil, "which", fake_which) - with pytest.raises(RuntimeError, match="clang and lld"): - _build_shared._check_toolchain_available("llvm") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_present_passes(self, monkeypatch): - monkeypatch.setattr(_build_shared.shutil, "which", lambda name: "/bin/" + name) - _build_shared._check_toolchain_available("llvm") - - @pytest.fixture def stamp(tmp_path, monkeypatch): - """Redirect the toolchain stamp to a scratch path.""" + """Redirect the toolchain stamp to a scratch path and reset the shared force flag.""" scratch = tmp_path / "build" / ".build-toolchain" monkeypatch.setattr(build_hooks, "_BUILD_TOOLCHAIN_STAMP", scratch) monkeypatch.setattr(_build_shared, "force_build_ext", False) @@ -403,23 +86,10 @@ def _write_stamp(stamp, toolchain): class TestBuildToolchainStamp: - """Tests for _check_build_toolchain() and record_build_toolchain().""" + """cuda.bindings stamps the toolchain name, through _check_build_toolchain() and record_build_toolchain(). - @pytest.mark.agent_authored(model="grok-4.6") - def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): - monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") - python_310 = build_hooks._abi_stamp_path(".build-toolchain") - monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") - python_311 = build_hooks._abi_stamp_path(".build-toolchain") - - assert python_310 != python_311 - assert python_310.name == ".build-toolchain.cpython-310-x86_64-linux-gnu.so" - assert python_311.name == ".build-toolchain.cpython-311-x86_64-linux-gnu.so" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_missing_stamp_forces_rebuild(self, stamp): - build_hooks._check_build_toolchain("gnu") - assert build_hooks.force_build_ext is True + The stamp-and-force protocol itself is tested by TestBuildKeyStamp. + """ @pytest.mark.agent_authored(model="glm-5.2") def test_same_toolchain_does_not_force(self, stamp): @@ -456,9 +126,76 @@ def test_record_writes_stamp(self, stamp): if _test_helpers_root.is_dir() and str(_test_helpers_root) not in sys.path: sys.path.insert(0, str(_test_helpers_root)) +from cuda_python_test_helpers.build_shared import ( + AbiStampPathMixin, + BuildKeyStampMixin, + CheckToolchainAvailableMixin, + DistutilsLinkerIntegrationMixin, + ForceBuildExtReexportMixin, + ResolveToolchainMixin, + WithCompilerMixin, + WithSccacheMixin, +) from cuda_python_test_helpers.cython_cache import POSIX_ONLY_CACHE, CythonAliasMixin, CythonCachePathMixin +class TestResolveToolchainShared(ResolveToolchainMixin): + build_shared = _build_shared + + +class TestWithSccache(WithSccacheMixin): + build_shared = _build_shared + + +class TestWithCompiler(WithCompilerMixin): + build_shared = _build_shared + + +class TestDistutilsLinkerIntegration(DistutilsLinkerIntegrationMixin): + build_shared = _build_shared + + +class TestCheckToolchainAvailable(CheckToolchainAvailableMixin): + build_shared = _build_shared + + +class TestAbiStampPath(AbiStampPathMixin): + build_shared = _build_shared + + +class TestBuildKeyStamp(BuildKeyStampMixin): + build_shared = _build_shared + + +class TestForceBuildExtReexport(ForceBuildExtReexportMixin): + build_hooks = build_hooks + build_shared = _build_shared + + +class TestResolveToolchain: + """What cuda.bindings chooses in its ``_resolve_toolchain`` wrapper.""" + + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_linux_flag_set(self, monkeypatch): + """c++14 (c++17 costs ~15% on launch benchmarks), plus -Wno-deprecated-declarations; no -Werror.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "-std=c++14" in cargs + assert "-Wno-deprecated-declarations" in cargs + assert "-Werror" not in cargs + + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_msvc_flag_set(self, monkeypatch): + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "/std:c++14" in cargs + assert "/WX" not in cargs + + class TestCythonCachePath(CythonCachePathMixin): """`_cython_cache_path` tests specific to cuda.bindings. diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index b914896c35f..7fb27386cbe 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -19,9 +19,7 @@ import builtins import importlib.util import os -import shlex import sys -import sysconfig import tempfile import threading import types @@ -35,8 +33,6 @@ import Cython # noqa: F401 import pytest import setuptools # noqa: F401 -from setuptools._distutils.ccompiler import new_compiler -from setuptools._distutils.sysconfig import customize_compiler from cuda.pathfinder import get_cuda_path_or_home @@ -229,17 +225,6 @@ def _write_stamp(stamp, config_key): class TestBuildConfigStamp: """Tests for _check_build_config() and record_build_config().""" - @pytest.mark.agent_authored(model="grok-4.6") - def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): - monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so") - python_310 = build_hooks._abi_stamp_path(".build-config") - monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so") - python_311 = build_hooks._abi_stamp_path(".build-config") - - assert python_310 != python_311 - assert python_310.name == ".build-config.cpython-310-x86_64-linux-gnu.so" - assert python_311.name == ".build-config.cpython-311-x86_64-linux-gnu.so" - @pytest.mark.agent_authored(model="glm-5.2") def test_missing_stamp_forces_rebuild(self, stamp): # No stamp means the last build's config is unknown, so rebuild. @@ -895,334 +880,97 @@ def fake_cythonize(ext_modules, **kwargs): assert captured["macros"] == {tuple(build_hooks._build_define_macros("13"))} -class TestResolveToolchain: - """_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN. - - The default toolchain (gnu on Linux, msvc on Windows) must reproduce the - previous build behavior exactly and must not touch CC/CXX/LDCXXSHARED, so an - externally-set compiler (e.g. the sccache wrapper in CI) survives. - """ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_does_not_touch_env(self, monkeypatch): - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - monkeypatch.delenv("LDCXXSHARED", raising=False) - name, cc, cxx, _cargs, _largs = build_hooks._resolve_toolchain() - if sys.platform == "win32": - assert name == "msvc" - assert cc is None and cxx is None - else: - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert "CC" not in os.environ and "CXX" not in os.environ - assert "LDCXXSHARED" not in os.environ - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_preserves_existing_cc(self, monkeypatch): - # An externally-set CC (e.g. sccache) must survive the default toolchain. - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache cc" - assert os.environ["CXX"] == "sccache c++" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_case_insensitive(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "LLVM") - name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert name == "llvm" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_invalid_value_raises(self, monkeypatch): - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "icc") - with pytest.raises(RuntimeError, match="not supported"): - build_hooks._resolve_toolchain() - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "llvm" - assert (cc, cxx) == ("clang", "clang++") - assert os.environ["CC"] == "clang" - assert os.environ["CXX"] == "clang++" - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - assert "-fuse-ld=lld" in largs - # clang rejects the gcc-only flags that gnu uses; they must be absent. - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="glm-5.2") - def test_gnu_sets_env_and_flags(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") - _fake_sysconfig(monkeypatch, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") - monkeypatch.delenv("CC", raising=False) - monkeypatch.delenv("CXX", raising=False) - monkeypatch.delenv("LDSHARED", raising=False) - name, cc, cxx, cargs, largs = build_hooks._resolve_toolchain() - assert name == "gnu" - assert (cc, cxx) == ("gcc", "g++") - assert os.environ["CC"] == "gcc" - assert os.environ["CXX"] == "g++" - assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" - assert "LDSHARED" not in os.environ - # Neither cuda.core nor cuda.bindings sets these gcc-only flags. - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - - @pytest.mark.agent_authored(model="grok-4.6") - def test_llvm_keeps_sccache_prefix(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - _name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain() - assert os.environ["CC"] == "sccache clang" - assert os.environ["CXX"] == "sccache clang++" - assert "LDSHARED" not in os.environ - # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" +# --------------------------------------------------------------------------- +# Cython cache path helper (workaround for cython/cython#7532) +# +# These tests cover the configuration-digest workaround in build_hooks.py. +# They can be deleted together with the `_cython_cache_path` helper once +# cython/cython#7532 is resolved in a released Cython version and +# cuda-python's minimum Cython version includes the fix. +# See https://github.com/cython/cython/issues/7532 - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - _fake_sysconfig(monkeypatch, LDCXXSHARED="g++ -shared -Wl,-O1") - monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): - if sys.platform == "win32": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - _fake_sysconfig(monkeypatch, LDSHARED="gcc -shared -Wl,-z,relro") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" - _fake_sysconfig(monkeypatch) - os.environ.pop("LDCXXSHARED") - build_hooks._resolve_toolchain() - assert os.environ["LDCXXSHARED"] == "clang++ -shared" +_test_helpers_root = Path(__file__).parents[2] / "cuda_python_test_helpers" +if _test_helpers_root.is_dir() and str(_test_helpers_root) not in sys.path: + sys.path.insert(0, str(_test_helpers_root)) - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_linux_opt_flag_set(self, monkeypatch): - """Linux opt build: -std=c++17, -O2; no gnu-only flags.""" - if sys.platform == "win32": - pytest.skip("Linux flags only") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain(debug=False) - assert "-std=c++17" in cargs - assert "-g0" in cargs - assert "-O2" in cargs - assert "-fpermissive" not in cargs - assert "-fno-var-tracking-assignments" not in cargs - assert "-Wl,--strip-all" in largs +from cuda_python_test_helpers.build_shared import ( + AbiStampPathMixin, + BuildKeyStampMixin, + CheckToolchainAvailableMixin, + DistutilsLinkerIntegrationMixin, + ForceBuildExtReexportMixin, + ResolveToolchainMixin, + WithCompilerMixin, + WithSccacheMixin, +) +from cuda_python_test_helpers.cython_cache import POSIX_ONLY_CACHE, CythonAliasMixin, CythonCachePathMixin - @pytest.mark.agent_authored(model="claude-sonnet-4-6") - def test_msvc_opt_flag_set(self, monkeypatch): - """MSVC opt build emits /std:c++17 and /O2.""" - if sys.platform != "win32": - pytest.skip("MSVC flags only on Windows") - monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) - _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) - assert "/std:c++17" in cargs - assert "/O2" in cargs +class TestResolveToolchainShared(ResolveToolchainMixin): + build_shared = _build_shared -class TestWithSccache: - """_with_sccache: keep a leading sccache token, swap the compiler.""" - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_sccache_and_swaps_compiler(self): - assert _build_shared._with_sccache("sccache cc", "clang") == "sccache clang" +class TestWithSccache(WithSccacheMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_absolute_sccache_path(self): - assert ( - _build_shared._with_sccache("/host/usr/local/bin/sccache cc", "clang") - == "/host/usr/local/bin/sccache clang" - ) - @pytest.mark.agent_authored(model="grok-4.6") - def test_bare_or_unrelated_cc_returns_compiler(self): - assert _build_shared._with_sccache("", "clang") == "clang" - assert _build_shared._with_sccache("gcc", "clang") == "clang" - assert _build_shared._with_sccache("ccache gcc", "clang") == "clang" +class TestWithCompiler(WithCompilerMixin): + build_shared = _build_shared -class TestWithCompiler: - """_with_compiler: replace the compiler executable, keep following flags.""" +class TestDistutilsLinkerIntegration(DistutilsLinkerIntegrationMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="grok-4.6") - def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): - assert ( - _build_shared._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") - == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" - ) - @pytest.mark.agent_authored(model="grok-4.6") - def test_compiler_only_command_returns_compiler(self): - assert _build_shared._with_compiler("g++", "clang++") == "clang++" - assert _build_shared._with_compiler("", "clang++") == "clang++" - assert _build_shared._with_compiler(None, "clang++") == "clang++" +class TestCheckToolchainAvailable(CheckToolchainAvailableMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_quoted_arguments_intact(self): - result = _build_shared._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") - assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_drops_launcher_before_compiler(self): - # setuptools takes the launcher from CXX; keeping a second copy here would - # leave a stray compiler argument on the link line. - assert _build_shared._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" +class TestAbiStampPath(AbiStampPathMixin): + build_shared = _build_shared - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_keeps_env_prefix(self): - assert ( - _build_shared._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") - == "env LIBRARY_PATH=/custom/lib clang++ -shared" - ) - # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env - assert ( - _build_shared._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") - == "env --unset=LD_LIBRARY_PATH clang++ -shared" - ) +class TestBuildKeyStamp(BuildKeyStampMixin): + build_shared = _build_shared -class TestDistutilsLinkerIntegration: - """The env set by _resolve_toolchain, as setuptools' distutils consumes it. - The tests above check os.environ; this checks the linker commands distutils - derives from it, which is what actually reaches the C++ link step. - """ +class TestForceBuildExtReexport(ForceBuildExtReexportMixin): + build_hooks = build_hooks + build_shared = _build_shared - @staticmethod - def _customized_compiler(): - compiler = new_compiler() - customize_compiler(compiler) - if not hasattr(compiler, "linker_so_cxx"): - pytest.skip("this setuptools' distutils has no linker_so_cxx") - return compiler - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") - if not sysconfig_ld: - pytest.skip("this Python has no LDCXXSHARED") - # Everything from the first flag on, including operands such as ``-B /path``. - tokens = shlex.split(sysconfig_ld) - first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) - expected_tail = tokens[first_flag:] - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[0] == "clang++" - assert linker[1 : 1 + len(expected_tail)] == expected_tail +class TestResolveToolchain: + """What cuda.core chooses in its ``_resolve_toolchain`` wrapper.""" - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:6] == [ - "clang++", - "-pthread", - "-B", - "/path/to/python_compiler_compat", - "-shared", - "-Wl,-rpath,/lib", - ] + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_linux_flag_set(self, monkeypatch): + """c++17 (structured bindings and if constexpr in cuda/core/_cpp/); warnings are not errors by default.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setattr(build_hooks, "WARNINGS_AS_ERRORS", False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "-std=c++17" in cargs + assert "-Werror" not in cargs - @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - build_hooks._resolve_toolchain() - linker = self._customized_compiler().linker_so_cxx - assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] - assert "g++" not in linker + @pytest.mark.agent_authored(model="claude-sonnet-4-6") + def test_msvc_flag_set(self, monkeypatch): + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setattr(build_hooks, "WARNINGS_AS_ERRORS", False) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert "/std:c++17" in cargs + assert "/WX" not in cargs @pytest.mark.agent_authored(model="claude-sonnet-5.5") - def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): - if sys.platform != "linux": - pytest.skip("gnu/llvm only valid on Linux") - monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") - monkeypatch.setenv("CC", "sccache cc") - monkeypatch.setenv("CXX", "sccache c++") - build_hooks._resolve_toolchain() - compiler = self._customized_compiler() - assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] - linker = compiler.linker_so_cxx - assert linker[0] == "clang++" - assert linker.count("clang++") == 1 - - -class TestCheckToolchainAvailable: - """_check_toolchain_available: fast, helpful failure when a tool is missing.""" - - @pytest.mark.agent_authored(model="glm-5.2") - def test_default_is_noop(self): - # The platform default never preflights. - _build_shared._check_toolchain_available("gnu") - _build_shared._check_toolchain_available("msvc") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): - def fake_which(name): - return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name - - monkeypatch.setattr(_build_shared.shutil, "which", fake_which) - with pytest.raises(RuntimeError, match="clang and lld"): - _build_shared._check_toolchain_available("llvm") - - @pytest.mark.agent_authored(model="glm-5.2") - def test_llvm_present_passes(self, monkeypatch): - monkeypatch.setattr(_build_shared.shutil, "which", lambda name: "/bin/" + name) - _build_shared._check_toolchain_available("llvm") - - -# --------------------------------------------------------------------------- -# Cython cache path helper (workaround for cython/cython#7532) -# -# These tests cover the configuration-digest workaround in build_hooks.py. -# They can be deleted together with the `_cython_cache_path` helper once -# cython/cython#7532 is resolved in a released Cython version and -# cuda-python's minimum Cython version includes the fix. -# See https://github.com/cython/cython/issues/7532 - - -_test_helpers_root = Path(__file__).parents[2] / "cuda_python_test_helpers" -if _test_helpers_root.is_dir() and str(_test_helpers_root) not in sys.path: - sys.path.insert(0, str(_test_helpers_root)) - -from cuda_python_test_helpers.cython_cache import POSIX_ONLY_CACHE, CythonAliasMixin, CythonCachePathMixin + def test_cuda_python_werror_makes_warnings_errors(self, monkeypatch): + """CUDA_PYTHON_WERROR=1 (read into WARNINGS_AS_ERRORS) reaches the shared flag set.""" + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setattr(build_hooks, "WARNINGS_AS_ERRORS", True) + _name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain(debug=False) + assert ("/WX" if sys.platform == "win32" else "-Werror") in cargs class TestCudaCoreCythonIncludePath: diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py new file mode 100644 index 00000000000..ac45266ca26 --- /dev/null +++ b/cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py @@ -0,0 +1,499 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared tests for the helpers in ``_build_shared.py``. + +The helpers live in ``cuda_bindings/_build_shared.py``, and +``cuda_core/_build_shared.py`` is a symlink to that file, so their behavior is +identical whichever backend loads them. The mixins here hold the tests of that +shared behavior. Each package's ``tests/test_build_hooks.py`` mixes them in +against the ``_build_shared`` module it loaded, by setting the class attribute +``build_shared``. + +Kept out of the mixins on purpose: + +- What each package decides: the C++ standard, ``tweak`` flags, whether + warnings are errors, and the key it stamps (the toolchain for cuda-bindings; + CUDA major, toolchain, debug and coverage for cuda-core). Those tests stay + in the package's own file. +- Anything that a normal wheel build already proves, such as the platform + default toolchain resolving, or the llvm preflight being a no-op for the + default. A regression there breaks every wheel build. + +Tests that change ``force_build_ext`` do so on ``build_shared`` itself. +``build_hooks`` re-exports the flag through a module ``__getattr__``, so +``monkeypatch.setattr(build_hooks, "force_build_ext", ...)`` would, on teardown, +leave a plain attribute on ``build_hooks`` that shadows the re-export for every +later test. +""" + +import os +import shlex +import sys +import sysconfig + +import pytest +from setuptools._distutils.ccompiler import new_compiler +from setuptools._distutils.sysconfig import customize_compiler + + +def _fake_sysconfig(monkeypatch, build_shared, **values): + """Pin sysconfig.get_config_var so linker-command assertions are exact.""" + monkeypatch.setattr(build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) + + +class ResolveToolchainMixin: + """``resolve_toolchain`` behavior that does not depend on a package's choices. + + Subclasses set ``build_shared`` (the loaded ``_build_shared`` module). The + ``cxx_std`` passed here is arbitrary: these tests exercise the environment, + name and shared flag mechanics. Each package's own tests assert the + standard and the tweaks that it chooses. + """ + + build_shared = None + cxx_std = 17 + + def _resolve(self, **kwargs): + return self.build_shared.resolve_toolchain(cxx_std=self.cxx_std, **kwargs) + + @pytest.mark.agent_authored(model="glm-5.2") + def test_default_does_not_touch_env(self, monkeypatch): + # The default toolchain must leave the compiler environment alone: an + # externally-set compiler (e.g. the sccache wrapper in CI) survives. + for name in ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED", "LDCXXSHARED"): + monkeypatch.delenv(name, raising=False) + self._resolve() + for name in ("CC", "CXX", "LDSHARED", "LDCXXSHARED"): + assert name not in os.environ + + @pytest.mark.agent_authored(model="glm-5.2") + def test_default_preserves_existing_cc(self, monkeypatch): + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + self._resolve() + assert os.environ["CC"] == "sccache cc" + assert os.environ["CXX"] == "sccache c++" + + @pytest.mark.agent_authored(model="glm-5.2") + def test_case_insensitive(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "LLVM") + name, _cc, _cxx, _cargs, _largs = self._resolve() + assert name == "llvm" + + @pytest.mark.agent_authored(model="glm-5.2") + def test_invalid_value_raises(self, monkeypatch): + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "icc") + with pytest.raises(RuntimeError, match="not supported"): + self._resolve() + + @pytest.mark.agent_authored(model="glm-5.2") + def test_llvm_sets_env_and_flags(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.delenv("CC", raising=False) + monkeypatch.delenv("CXX", raising=False) + monkeypatch.delenv("LDSHARED", raising=False) + name, cc, cxx, cargs, largs = self._resolve() + assert name == "llvm" + assert (cc, cxx) == ("clang", "clang++") + assert os.environ["CC"] == "clang" + assert os.environ["CXX"] == "clang++" + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ + assert "-fuse-ld=lld" in largs + # clang rejects the gcc-only flags that gnu used to use; they must be absent. + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + + @pytest.mark.agent_authored(model="glm-5.2") + def test_gnu_sets_env_and_flags(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="x86_64-linux-gnu-g++ -shared -Wl,-O1") + monkeypatch.delenv("CC", raising=False) + monkeypatch.delenv("CXX", raising=False) + monkeypatch.delenv("LDSHARED", raising=False) + name, cc, cxx, cargs, _largs = self._resolve() + assert name == "gnu" + assert (cc, cxx) == ("gcc", "g++") + assert os.environ["CC"] == "gcc" + assert os.environ["CXX"] == "g++" + assert os.environ["LDCXXSHARED"] == "g++ -shared -Wl,-O1" + assert "LDSHARED" not in os.environ + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + + @pytest.mark.agent_authored(model="grok-4.6") + def test_llvm_keeps_sccache_prefix(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + self._resolve() + assert os.environ["CC"] == "sccache clang" + assert os.environ["CXX"] == "sccache clang++" + assert "LDSHARED" not in os.environ + # The launcher prefixes CC/CXX only; the shared linker command is the bare compiler. + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-O1" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_prefers_env_ldcxxshared(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + _fake_sysconfig(monkeypatch, self.build_shared, LDCXXSHARED="g++ -shared -Wl,-O1") + monkeypatch.setenv("LDCXXSHARED", "g++ -shared -Wl,-rpath,/user/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-rpath,/user/lib" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_explicit_toolchain_falls_back_to_ldshared_then_shared_flag(self, monkeypatch): + if sys.platform == "win32": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + _fake_sysconfig(monkeypatch, self.build_shared, LDSHARED="gcc -shared -Wl,-z,relro") + self._resolve() + assert os.environ["LDCXXSHARED"] == "clang++ -shared -Wl,-z,relro" + _fake_sysconfig(monkeypatch, self.build_shared) + os.environ.pop("LDCXXSHARED") + self._resolve() + assert os.environ["LDCXXSHARED"] == "clang++ -shared" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linux_opt_flag_set(self, monkeypatch): + """The one Linux opt flag set: -std, -g0 -O2, stripped link; no -O3 or gcc-only flags.""" + if sys.platform == "win32": + pytest.skip("Linux flags only") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, largs = self._resolve(debug=False) + assert f"-std=c++{self.cxx_std}" in cargs + assert "-g0" in cargs + assert "-O2" in cargs + assert "-O3" not in cargs + assert "-fpermissive" not in cargs + assert "-fno-var-tracking-assignments" not in cargs + assert "-Wl,--strip-all" in largs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_msvc_opt_flag_set(self, monkeypatch): + """Modern setuptools no longer forces /Ox, so /O2 must be emitted explicitly.""" + if sys.platform != "win32": + pytest.skip("MSVC flags only on Windows") + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = self._resolve(debug=False) + assert f"/std:c++{self.cxx_std}" in cargs + assert "/O2" in cargs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_warnings_as_errors_is_opt_in(self, monkeypatch): + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = self._resolve() + assert "-Werror" not in cargs + assert "/WX" not in cargs + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_warnings_as_errors_flags_per_platform(self, monkeypatch): + # A wheel build passes without these flags, so a refactor could drop + # them and silently disable the Werror gate that CI relies on. + monkeypatch.delenv("CUDA_PYTHON_TOOLCHAIN", raising=False) + _name, _cc, _cxx, cargs, _largs = self._resolve(warnings_as_errors=True) + if sys.platform == "win32": + assert {"/WX", "/wd4551", "/wd4244"} <= set(cargs) + else: + assert "-Werror" in cargs + + +class WithSccacheMixin: + """``_with_sccache``: keep a leading sccache token, swap the compiler.""" + + build_shared = None + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_sccache_and_swaps_compiler(self): + assert self.build_shared._with_sccache("sccache cc", "clang") == "sccache clang" + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_absolute_sccache_path(self): + assert ( + self.build_shared._with_sccache("/host/usr/local/bin/sccache cc", "clang") + == "/host/usr/local/bin/sccache clang" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_bare_or_unrelated_cc_returns_compiler(self): + assert self.build_shared._with_sccache("", "clang") == "clang" + assert self.build_shared._with_sccache("gcc", "clang") == "clang" + assert self.build_shared._with_sccache("ccache gcc", "clang") == "clang" + + +class WithCompilerMixin: + """``_with_compiler``: replace the compiler executable, keep the following flags.""" + + build_shared = None + + @pytest.mark.agent_authored(model="grok-4.6") + def test_keeps_flags_that_were_part_of_sysconfig_cxx(self): + assert ( + self.build_shared._with_compiler("g++ -pthread -B /compat -shared -Wl,-rpath,/lib", "clang++") + == "clang++ -pthread -B /compat -shared -Wl,-rpath,/lib" + ) + + @pytest.mark.agent_authored(model="grok-4.6") + def test_compiler_only_command_returns_compiler(self): + assert self.build_shared._with_compiler("g++", "clang++") == "clang++" + assert self.build_shared._with_compiler("", "clang++") == "clang++" + assert self.build_shared._with_compiler(None, "clang++") == "clang++" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_quoted_arguments_intact(self): + result = self.build_shared._with_compiler("g++ -Wl,-rpath='/a b' -shared", "clang++") + assert shlex.split(result) == ["clang++", "-Wl,-rpath=/a b", "-shared"] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_drops_launcher_before_compiler(self): + # setuptools takes the launcher from CXX; keeping a second copy here would + # leave a stray compiler argument on the link line. + assert self.build_shared._with_compiler("ccache g++ -shared", "clang++") == "clang++ -shared" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_keeps_env_prefix(self): + assert ( + self.build_shared._with_compiler("env LIBRARY_PATH=/custom/lib g++ -shared", "clang++") + == "env LIBRARY_PATH=/custom/lib clang++ -shared" + ) + # env long options (--unset=VAR) also treated as prefix, same as setuptools' _split_env + assert ( + self.build_shared._with_compiler("env --unset=LD_LIBRARY_PATH g++ -shared", "clang++") + == "env --unset=LD_LIBRARY_PATH clang++ -shared" + ) + + +class DistutilsLinkerIntegrationMixin: + """The env set by ``resolve_toolchain``, as setuptools' distutils consumes it. + + ``ResolveToolchainMixin`` checks os.environ; this checks the linker commands + distutils derives from it, which is what reaches the C++ link step. + """ + + build_shared = None + cxx_std = 17 + + def _resolve(self): + return self.build_shared.resolve_toolchain(cxx_std=self.cxx_std) + + @staticmethod + def _customized_compiler(): + compiler = new_compiler() + customize_compiler(compiler) + if not hasattr(compiler, "linker_so_cxx"): + pytest.skip("this setuptools' distutils has no linker_so_cxx") + return compiler + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_swaps_compiler_and_keeps_sysconfig_flags(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + sysconfig_ld = sysconfig.get_config_var("LDCXXSHARED") + if not sysconfig_ld: + pytest.skip("this Python has no LDCXXSHARED") + # Everything from the first flag on, including operands such as ``-B /path``. + tokens = shlex.split(sysconfig_ld) + first_flag = next((i for i, tok in enumerate(tokens) if tok.startswith("-")), len(tokens)) + expected_tail = tokens[first_flag:] + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + linker = self._customized_compiler().linker_so_cxx + assert linker[0] == "clang++" + assert linker[1 : 1 + len(expected_tail)] == expected_tail + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_split_option_operands(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "g++ -pthread -B /path/to/python_compiler_compat -shared -Wl,-rpath,/lib") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + linker = self._customized_compiler().linker_so_cxx + assert linker[:6] == [ + "clang++", + "-pthread", + "-B", + "/path/to/python_compiler_compat", + "-shared", + "-Wl,-rpath,/lib", + ] + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_linker_so_cxx_keeps_env_prefix(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("LDCXXSHARED", "env LIBRARY_PATH=/custom/lib g++ -shared") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + self._resolve() + linker = self._customized_compiler().linker_so_cxx + assert linker[:3] == ["env", "LIBRARY_PATH=/custom/lib", "clang++"] + assert "g++" not in linker + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_sccache_does_not_duplicate_compiler_in_cxx_linker(self, monkeypatch): + if sys.platform != "linux": + pytest.skip("gnu/llvm only valid on Linux") + monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm") + monkeypatch.setenv("CC", "sccache cc") + monkeypatch.setenv("CXX", "sccache c++") + self._resolve() + compiler = self._customized_compiler() + assert compiler.compiler_cxx[:2] == ["sccache", "clang++"] + linker = compiler.linker_so_cxx + assert linker[0] == "clang++" + assert linker.count("clang++") == 1 + + +class CheckToolchainAvailableMixin: + """``_check_toolchain_available``: a helpful failure when an llvm tool is missing. + + The platform default never preflights, and a wheel build proves that, so + only the error message and the llvm-tools-present case are tested. + """ + + build_shared = None + + @pytest.mark.agent_authored(model="glm-5.2") + def test_llvm_missing_tool_lists_install_hint(self, monkeypatch): + def fake_which(name): + return None if name in ("clang", "clang++", "ld.lld") else "/bin/" + name + + monkeypatch.setattr(self.build_shared.shutil, "which", fake_which) + with pytest.raises(RuntimeError, match="clang and lld"): + self.build_shared._check_toolchain_available("llvm") + + @pytest.mark.agent_authored(model="glm-5.2") + def test_llvm_present_passes(self, monkeypatch): + monkeypatch.setattr(self.build_shared.shutil, "which", lambda name: "/bin/" + name) + self.build_shared._check_toolchain_available("llvm") + + +class AbiStampPathMixin: + """``_abi_stamp_path`` scopes stamp files by Python's EXT_SUFFIX. + + The stem is arbitrary: each package uses its own at run time + (``.build-toolchain`` for cuda-bindings, ``.build-config`` for cuda-core), + but the mechanism is shared. + """ + + build_shared = None + + @pytest.mark.agent_authored(model="grok-4.6") + def test_stamp_path_is_scoped_to_extension_abi(self, monkeypatch): + monkeypatch.setattr( + self.build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-310-x86_64-linux-gnu.so" + ) + python_310 = self.build_shared._abi_stamp_path(".build-test") + monkeypatch.setattr( + self.build_shared.sysconfig, "get_config_var", lambda _name: ".cpython-311-x86_64-linux-gnu.so" + ) + python_311 = self.build_shared._abi_stamp_path(".build-test") + + assert python_310 != python_311 + assert python_310.name == ".build-test.cpython-310-x86_64-linux-gnu.so" + assert python_311.name == ".build-test.cpython-311-x86_64-linux-gnu.so" + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_missing_ext_suffix_is_an_error(self, monkeypatch): + monkeypatch.setattr(self.build_shared.sysconfig, "get_config_var", lambda _name: None) + with pytest.raises(RuntimeError, match="EXT_SUFFIX"): + self.build_shared._abi_stamp_path(".build-test") + + +class BuildKeyStampMixin: + """``check_build_key`` and ``record_build_key``: the stamp-and-force protocol. + + The tests use a scratch stamp, so they do not depend on what key a package + stamps. Those keys are tested in the package's own file. + """ + + build_shared = None + + @pytest.fixture(autouse=True) + def _reset_force_build_ext(self, monkeypatch): + monkeypatch.setattr(self.build_shared, "force_build_ext", False) + + @pytest.fixture + def stamp_file(self, tmp_path): + return tmp_path / "build" / ".build-test" + + @pytest.mark.agent_authored(model="glm-5.2") + def test_missing_stamp_forces_rebuild(self, stamp_file): + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + assert self.build_shared.force_build_ext is True + + @pytest.mark.agent_authored(model="glm-5.2") + def test_same_key_does_not_force(self, stamp_file): + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + assert self.build_shared.force_build_ext is False + + @pytest.mark.agent_authored(model="glm-5.2") + def test_changed_key_forces_rebuild(self, stamp_file): + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "llvm", "Toolchain") + assert self.build_shared.force_build_ext is True + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_a_later_match_does_not_clear_the_flag(self, stamp_file): + # Once one check asks for a full rebuild, a second, unchanged check must not retract it. + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "gnu", "Toolchain") + assert self.build_shared.force_build_ext is True + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_message_names_the_previous_and_the_new_key(self, stamp_file, capsys): + self.build_shared.record_build_key(stamp_file, "gnu") + self.build_shared.check_build_key(stamp_file, "llvm", "Toolchain") + out = capsys.readouterr().out + assert "Toolchain of last build: gnu (building llvm); forcing a full rebuild" in out + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_record_creates_the_directory_and_writes_the_key(self, stamp_file): + assert not stamp_file.parent.exists() + self.build_shared.record_build_key(stamp_file, "cu13-gnu-opt") + assert stamp_file.read_text(encoding="utf-8") == "cu13-gnu-opt\n" + + +class ForceBuildExtReexportMixin: + """``build_hooks.force_build_ext`` is a live view of ``build_shared.force_build_ext``. + + setup.py reads the flag as ``build_hooks.force_build_ext``. The flag is + owned by ``_build_shared``; ``build_hooks`` re-exports it. Subclasses set + both ``build_hooks`` and ``build_shared``. + """ + + build_hooks = None + build_shared = None + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_build_hooks_follows_the_shared_flag(self, monkeypatch): + monkeypatch.setattr(self.build_shared, "force_build_ext", True) + assert self.build_hooks.force_build_ext is True + monkeypatch.setattr(self.build_shared, "force_build_ext", False) + assert self.build_hooks.force_build_ext is False + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_build_hooks_holds_no_copy_of_the_flag(self): + # A plain attribute would shadow the module __getattr__ and go stale. + assert "force_build_ext" not in vars(self.build_hooks) + + @pytest.mark.agent_authored(model="claude-sonnet-5.5") + def test_other_names_still_raise_attribute_error(self): + with pytest.raises(AttributeError, match="no_such_name"): + self.build_hooks.no_such_name # noqa: B018 From 3046139b4b0248265f3905523328c4b7ccbe6289 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 09:41:31 -0700 Subject: [PATCH 7/9] docs: point the source-install notes at Development on Windows cuda_core/_build_shared.py is a symlink into cuda_bindings, so a Windows source build fails confusingly when git symlinks were off at clone time. CONTRIBUTING.md now documents the setting; link to it from the cuda-bindings and cuda-core source-install notes. --- cuda_bindings/docs/source/install.rst | 2 +- cuda_core/docs/source/install.rst | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cuda_bindings/docs/source/install.rst b/cuda_bindings/docs/source/install.rst index b89feb4aa89..73b32eedcac 100644 --- a/cuda_bindings/docs/source/install.rst +++ b/cuda_bindings/docs/source/install.rst @@ -126,7 +126,7 @@ Requirements [^2]: The CUDA Runtime static library (``libcudart_static.a`` on Linux, ``cudart_static.lib`` on Windows) is part of the CUDA Toolkit. If using conda packages, it is contained in the ``cuda-cudart-static`` package. -[^3]: The version is derived from git tags via ``setuptools-scm``, so the clone must include tags reaching back to at least the latest ``v*`` tag. Clone with ``git clone https://github.com/NVIDIA/cuda-python.git``; do not use ``--depth`` or ``--no-tags``, since a shallow clone builds without error but produces a bogus version such as ``0.1.dev1+g0d22cb444``. See `Cloning the repository `_ for details and recovery steps. +[^3]: The version is derived from git tags via ``setuptools-scm``, so the clone must include tags reaching back to at least the latest ``v*`` tag. Clone with ``git clone https://github.com/NVIDIA/cuda-python.git``; do not use ``--depth`` or ``--no-tags``, since a shallow clone builds without error but produces a bogus version such as ``0.1.dev1+g0d22cb444``. See `Cloning the repository `_ for details and recovery steps. Windows contributors building outside of WSL should also see `Development on Windows `_ for the git-symlink configuration that must be set *before* cloning. Source builds require that the provided CUDA headers are of the same major.minor version as the ``cuda.bindings`` you're trying to build. Despite this requirement, note that the minor version compatibility is still maintained. The build checks the header before it compiles anything. A mismatch stops the build with a message that names the ``cuda.h`` it found and the version this source tree needs. Use the ``CUDA_PATH`` (or ``CUDA_HOME``) environment variable to specify the location of your headers. If both are set, ``CUDA_PATH`` takes precedence. For example, if your headers are located in ``/usr/local/cuda/include``, then you should set ``CUDA_PATH`` with: diff --git a/cuda_core/docs/source/install.rst b/cuda_core/docs/source/install.rst index 320d8179982..fbb4f7dd836 100644 --- a/cuda_core/docs/source/install.rst +++ b/cuda_core/docs/source/install.rst @@ -178,4 +178,8 @@ A source build has two requirements. See :ref:`cuda-core-bindings-floor`. error but produces a bogus version such as ``0.1.dev1+g0d22cb444``. See `Cloning the repository `_ - for details and recovery steps. + for details and recovery steps. Windows contributors building outside of + WSL should also see + `Development on Windows + `_ + for the git-symlink configuration that must be set *before* cloning. From b4ba4b81fa3036d8d9439beb097f67a7c6ee719b Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 09:46:25 -0700 Subject: [PATCH 8/9] build: move the CUDA path lookup into _build_shared.py _import_get_cuda_path_or_home() and _get_cuda_path() were byte-identical copies in the two build_hooks.py files, held together by "keep in sync" comments. Move them into the shared helper module and import them from both backends. The tests that patch build_hooks._get_cuda_path or call its cache_clear() keep working: the name is bound in each build_hooks module, and the cached function object is the same. --- cuda_bindings/_build_shared.py | 52 +++++++++++++++++++++++++++++++++- cuda_bindings/build_hooks.py | 48 +------------------------------ cuda_core/build_hooks.py | 49 ++------------------------------ 3 files changed, 54 insertions(+), 95 deletions(-) diff --git a/cuda_bindings/_build_shared.py b/cuda_bindings/_build_shared.py index 9ec331204ee..9fef4c91b5c 100644 --- a/cuda_bindings/_build_shared.py +++ b/cuda_bindings/_build_shared.py @@ -17,7 +17,7 @@ ``resolve_toolchain``: the C++ standard, warnings-as-errors, and an optional ``tweak`` hook for flags a single package needs. -Besides the toolchain, this module owns the machinery that both backends use +Besides the toolchain, this module owns the CUDA path lookup and the machinery that both backends use around cythonize and build_ext: the opt-in Cython generated-source cache and the build stamps that force a rebuild when the build configuration changed. @@ -26,6 +26,7 @@ """ import contextlib +import functools import hashlib import os import shlex @@ -36,6 +37,55 @@ from pathlib import Path from warnings import warn +# ----------------------------------------------------------------------- +# CUDA path + + +def _import_get_cuda_path_or_home(): + """Import get_cuda_path_or_home, working around PEP 517 namespace shadowing. + + See https://github.com/NVIDIA/cuda-python/issues/1824 for why this helper is needed. + """ + try: + import cuda.pathfinder + except ModuleNotFoundError as exc: + if exc.name not in ("cuda", "cuda.pathfinder"): + raise + try: + import cuda + except ModuleNotFoundError: + cuda = None + + for p in sys.path: + sp_cuda = Path(p) / "cuda" + if (sp_cuda / "pathfinder").is_dir(): + cuda.__path__ = list(cuda.__path__) + [str(sp_cuda)] + break + else: + raise ModuleNotFoundError( + "cuda-pathfinder is not installed in the build environment. " + "Ensure 'cuda-pathfinder>=1.5' is in build-system.requires." + ) + import cuda.pathfinder + + pathfinder_dir = Path(cuda.pathfinder.__file__).parent + print( + f"Using cuda-pathfinder {cuda.pathfinder.__version__} from {pathfinder_dir}", + file=sys.stderr, + ) + return cuda.pathfinder.get_cuda_path_or_home + + +@functools.cache +def _get_cuda_path() -> str: + get_cuda_path_or_home = _import_get_cuda_path_or_home() + cuda_path = get_cuda_path_or_home() + if not cuda_path: + raise RuntimeError("Environment variable CUDA_PATH or CUDA_HOME is not set") + print("CUDA path:", cuda_path) + return cuda_path + + # ----------------------------------------------------------------------- # Toolchain selection diff --git a/cuda_bindings/build_hooks.py b/cuda_bindings/build_hooks.py index af56bec3aab..e49cb99d48c 100644 --- a/cuda_bindings/build_hooks.py +++ b/cuda_bindings/build_hooks.py @@ -9,7 +9,6 @@ import atexit import contextlib -import functools import glob import os import re @@ -28,6 +27,7 @@ _abi_stamp_path, _check_toolchain_available, _cython_cache_path, + _get_cuda_path, _resolve_toolchain_name, _stable_cython_alias, check_build_key, @@ -67,52 +67,6 @@ def __getattr__(name): _INSTALL_URL = "https://nvidia.github.io/cuda-python/cuda-bindings/latest/install.html#installing-from-source" -# Please keep in sync with the copy in cuda_core/build_hooks.py. -def _import_get_cuda_path_or_home(): - """Import get_cuda_path_or_home, working around PEP 517 namespace shadowing. - - See https://github.com/NVIDIA/cuda-python/issues/1824 for why this helper is needed. - """ - try: - import cuda.pathfinder - except ModuleNotFoundError as exc: - if exc.name not in ("cuda", "cuda.pathfinder"): - raise - try: - import cuda - except ModuleNotFoundError: - cuda = None - - for p in sys.path: - sp_cuda = Path(p) / "cuda" - if (sp_cuda / "pathfinder").is_dir(): - cuda.__path__ = list(cuda.__path__) + [str(sp_cuda)] - break - else: - raise ModuleNotFoundError( - "cuda-pathfinder is not installed in the build environment. " - "Ensure 'cuda-pathfinder>=1.5' is in build-system.requires." - ) - import cuda.pathfinder - - pathfinder_dir = Path(cuda.pathfinder.__file__).parent - print( - f"Using cuda-pathfinder {cuda.pathfinder.__version__} from {pathfinder_dir}", - file=sys.stderr, - ) - return cuda.pathfinder.get_cuda_path_or_home - - -@functools.cache -def _get_cuda_path() -> str: - get_cuda_path_or_home = _import_get_cuda_path_or_home() - cuda_path = get_cuda_path_or_home() - if not cuda_path: - raise RuntimeError("Environment variable CUDA_PATH or CUDA_HOME is not set") - print("CUDA path:", cuda_path) - return cuda_path - - # ----------------------------------------------------------------------- # CUDA header check diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index 824b0a90fe2..529a0f548c6 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -29,6 +29,7 @@ _abi_stamp_path, _check_toolchain_available, _cython_cache_path, + _get_cuda_path, _stable_cython_alias, check_build_key, record_build_key, @@ -60,46 +61,10 @@ def __getattr__(name): WARNINGS_AS_ERRORS = bool(int(os.environ.get("CUDA_PYTHON_WERROR", "0"))) -# Please keep in sync with the copy in cuda_bindings/build_hooks.py. -def _import_get_cuda_path_or_home(): - """Import get_cuda_path_or_home, working around PEP 517 namespace shadowing. - - See https://github.com/NVIDIA/cuda-python/issues/1824 for why this helper is needed. - """ - try: - import cuda.pathfinder - except ModuleNotFoundError as exc: - if exc.name not in ("cuda", "cuda.pathfinder"): - raise - try: - import cuda - except ModuleNotFoundError: - cuda = None - - for p in sys.path: - sp_cuda = Path(p) / "cuda" - if (sp_cuda / "pathfinder").is_dir(): - cuda.__path__ = list(cuda.__path__) + [str(sp_cuda)] - break - else: - raise ModuleNotFoundError( - "cuda-pathfinder is not installed in the build environment. " - "Ensure 'cuda-pathfinder>=1.5' is in build-system.requires." - ) - import cuda.pathfinder - - pathfinder_dir = Path(cuda.pathfinder.__file__).parent - print( - f"Using cuda-pathfinder {cuda.pathfinder.__version__} from {pathfinder_dir}", - file=sys.stderr, - ) - return cuda.pathfinder.get_cuda_path_or_home - - def _import_cuda_bindings(): """Import cuda.bindings and work around PEP 517 namespace shadowing. - The problem and the repair are the same as in _import_get_cuda_path_or_home(). + The problem and the repair are the same as in _build_shared._import_get_cuda_path_or_home(). See https://github.com/NVIDIA/cuda-python/issues/1824. In an isolated build, the project's own ``cuda/`` directory is the whole ``cuda`` namespace. The cuda-bindings that pip installed into the build environment is not importable @@ -142,16 +107,6 @@ def _installed_cuda_bindings() -> tuple: return bindings.__version__, int(driver.CUDA_VERSION) -@functools.cache -def _get_cuda_path() -> str: - get_cuda_path_or_home = _import_get_cuda_path_or_home() - cuda_path = get_cuda_path_or_home() - if not cuda_path: - raise RuntimeError("Environment variable CUDA_PATH or CUDA_HOME is not set") - print("CUDA path:", cuda_path) - return cuda_path - - _PACKAGE_DIR = Path(__file__).parent / "cuda" / "core" _PYPROJECT_PATH = Path(__file__).parent / "pyproject.toml" From afaf4e8971f84e6cf0b694646b3a13a03e241a9e Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 2 Oct 2026 10:08:46 -0700 Subject: [PATCH 9/9] ci: enable git symlinks on remaining Windows checkouts coverage.yml and precommit-windows still cloned without core.symlinks=true, so cuda_core/_build_shared.py would land as a text stub. Drop the unused _fake_sysconfig leftovers from the mixin move. --- .github/workflows/ci.yml | 3 +++ .github/workflows/coverage.yml | 6 ++++++ cuda_bindings/tests/test_build_hooks.py | 8 +------- cuda_core/tests/test_build_hooks.py | 5 ----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15924c3073d..865dc9fe65a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -633,6 +633,9 @@ jobs: permissions: contents: read steps: + - name: Enable Git symlinks (Windows, must precede checkout) + run: git config --global core.symlinks true + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ad23d0e48a4..6dc6c3ff877 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -232,6 +232,9 @@ jobs: CUDA_PYTHON_COVERAGE: "1" CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} steps: + - name: Enable Git symlinks (Windows, must precede checkout) + run: git config --global core.symlinks true + - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -348,6 +351,9 @@ jobs: run: shell: bash --noprofile --norc -xeuo pipefail {0} steps: + - name: Enable Git symlinks (Windows, must precede checkout) + run: git config --global core.symlinks true + - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/cuda_bindings/tests/test_build_hooks.py b/cuda_bindings/tests/test_build_hooks.py index 182f4d390c3..b45b58fa102 100644 --- a/cuda_bindings/tests/test_build_hooks.py +++ b/cuda_bindings/tests/test_build_hooks.py @@ -4,8 +4,7 @@ """Tests for cuda_bindings/build_hooks.py build infrastructure. -Mirrors the toolchain tests in cuda_core/tests/test_build_hooks.py. These -tests do NOT require cuda.bindings to be built/installed since they test +These tests do NOT require cuda.bindings to be built/installed since they test build-time infrastructure. Run with --noconftest to avoid loading conftest.py which imports cuda.bindings modules: @@ -65,11 +64,6 @@ def _isolate_toolchain_env(): os.environ.update(original) -def _fake_sysconfig(monkeypatch, **values): - """Pin sysconfig.get_config_var so linker-command assertions are exact.""" - monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) - - @pytest.fixture def stamp(tmp_path, monkeypatch): """Redirect the toolchain stamp to a scratch path and reset the shared force flag.""" diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 7fb27386cbe..7c6dd2d9cc9 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -75,11 +75,6 @@ def _isolate_toolchain_env(): os.environ.update(original) -def _fake_sysconfig(monkeypatch, **values): - """Pin sysconfig.get_config_var so linker-command assertions are exact.""" - monkeypatch.setattr(_build_shared.sysconfig, "get_config_var", lambda name: values.get(name)) - - @pytest.mark.agent_authored(model="gpt-5.6") def test_cuda_path_is_resolved_before_importing_bindings(monkeypatch): """PEP 517 namespace repair runs before cuda.bindings is imported."""