from __future__ import annotations import importlib.util import os import platform from pathlib import Path import sys from setuptools import Extension, setup from setuptools.command.build_clib import build_clib from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension, ROCM_HOME ROOT = Path(__file__).parent KERNEL_INCLUDE = str(ROOT / "python" / "freetoken" / "kernel" / "csrc" / "include") def _check_toolchain() -> None: path = ROOT / "python" / "freetoken" / "kernel" / "_toolchain.py" spec = importlib.util.spec_from_file_location("_freetoken_toolchain", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.check_nvcc_matches_torch() def _is_rocm() -> bool: import torch return getattr(torch.version, "hip", None) is not None def _rocm_paths() -> tuple[list[str], list[str], str]: candidates: list[Path] = [] if os.getenv("ROCM_HOME"): candidates.append(Path(os.environ["ROCM_HOME"])) if ROCM_HOME: candidates.append(Path(ROCM_HOME)) # ROCm 7.14 PyTorch images ship the SDK as a Python package instead of # installing it at /opt/rocm. spec = importlib.util.find_spec("_rocm_sdk_core") if spec and spec.submodule_search_locations: candidates.append(Path(next(iter(spec.submodule_search_locations)))) candidates.append(Path("/opt/rocm")) for rocm_home in dict.fromkeys(candidates): include_dir = rocm_home / "include" library_dir = rocm_home / "lib" if not (include_dir / "hip" / "hip_runtime.h").exists(): continue if (library_dir / "libamdhip64.so").exists(): return [str(include_dir)], [str(library_dir)], "amdhip64" versioned = sorted(library_dir.glob("libamdhip64.so.*")) if versioned: return [str(include_dir)], [str(library_dir)], f":{versioned[-1].name}" searched = ", ".join(str(path) for path in dict.fromkeys(candidates)) raise RuntimeError( "A ROCm SDK with HIP headers and libamdhip64 is required to build on ROCm; " f"searched: {searched}. Set ROCM_HOME to override." ) def _cuda_runtime_paths() -> tuple[list[str], list[str]]: if CUDA_HOME is None: raise RuntimeError( "CUDA_HOME is required to build freetoken.kernel._pinned_tensor " "because it links against the CUDA runtime API." ) cuda_home = Path(CUDA_HOME) library_dirs = [str(cuda_home / "lib64")] if (cuda_home / "lib").exists(): library_dirs.append(str(cuda_home / "lib")) return [str(cuda_home / "include")], library_dirs IS_ROCM = _is_rocm() if IS_ROCM: runtime_include_dirs, runtime_library_dirs, runtime_lib = _rocm_paths() runtime_link_args = [f"-Wl,-rpath,{runtime_library_dirs[0]}"] # Host sources. BuildExtension supplies the ROCm # platform defines to the C++ compiler; offload architecture flags belong on # HIP device sources and would be rejected by the host compiler here. extra_compile = ["-O3", "-std=c++20"] # Device sources (.cu, built by hipcc); BuildExtension adds the offload architectures. device_compile = ["-O3", "-std=c++20"] else: runtime_include_dirs, runtime_library_dirs = _cuda_runtime_paths() runtime_lib = "cudart" runtime_link_args = [] extra_compile = ["-O3", "-std=c++20"] # Device sources (.cu, built by nvcc): SASS for each major arch, which also runs on the newer minors of that major, # plus PTX for the oldest arch nvcc 13 supports, which the driver builds for any other GPU at load. # Naming archs also stops torch from picking them, which fails on a build machine without a GPU. device_compile = ["-O3", "-std=c++20"] + [f"-gencode=arch=compute_{a},code=sm_{a}" for a in (75, 80, 90, 100, 120)] device_compile.append("-gencode=arch=compute_75,code=compute_75") _check_toolchain() # ggml-cpu's row kernels for GGUF experts on the CPU MoE executor: one module per x86 variant of llama.cpp's GGML_CPU_ALL_VARIANTS (ggml/src/CMakeLists.txt), freetoken.kernel.ggml_cpu._. # freetoken.kernel.ggml_cpu.select() imports the best one this CPU runs; each module's entry is built without the variant's ISA flags so that check is safe on any CPU. GGML_DIR = "python/freetoken/kernel/csrc/ggml" GGML_CPU = platform.machine().lower() in ("x86_64", "amd64") _HASWELL = ("SSE42", "AVX", "F16C", "FMA", "AVX2", "BMI2") GGML_CPU_VARIANTS = { "x64": (), "sse42": ("SSE42",), "sandybridge": ("SSE42", "AVX"), "ivybridge": ("SSE42", "AVX", "F16C"), "piledriver": ("SSE42", "AVX", "F16C", "FMA"), "haswell": _HASWELL, "skylakex": (*_HASWELL, "AVX512"), "cannonlake": (*_HASWELL, "AVX512", "AVX512_VBMI"), "cascadelake": (*_HASWELL, "AVX512", "AVX512_VNNI"), "icelake": (*_HASWELL, "AVX512", "AVX512_VBMI", "AVX512_VNNI"), "cooperlake": (*_HASWELL, "AVX512", "AVX512_VNNI", "AVX512_BF16"), "zen4": (*_HASWELL, "AVX512", "AVX512_VBMI", "AVX512_VNNI", "AVX512_BF16"), "alderlake": (*_HASWELL, "AVX_VNNI"), "sapphirerapids": (*_HASWELL, "AVX512", "AVX512_VBMI", "AVX512_VNNI", "AVX512_BF16", "AMX_TILE", "AMX_INT8"), } _GGML_GCC_FLAGS = { "SSE42": ["-msse4.2"], "F16C": ["-mf16c"], "FMA": ["-mfma"], "BMI2": ["-mbmi2"], "AVX": ["-mavx"], "AVX2": ["-mavx2"], "AVX_VNNI": ["-mavxvnni"], "AVX512": ["-mavx512f", "-mavx512cd", "-mavx512vl", "-mavx512dq", "-mavx512bw"], "AVX512_VBMI": ["-mavx512vbmi"], "AVX512_VNNI": ["-mavx512vnni"], "AVX512_BF16": ["-mavx512bf16"], "AMX_TILE": ["-mamx-tile"], "AMX_INT8": ["-mamx-int8"], } def _ggml_variant_flags(feats: tuple[str, ...]) -> tuple[list[str], list[str]]: """(ISA flags, macros) of one variant, as upstream's ggml-cpu/CMakeLists.txt sets them; the macros also tell cpu-feats.cpp what to check.""" if sys.platform != "win32": return [f for feat in feats for f in _GGML_GCC_FLAGS[feat]], [f"GGML_{feat}" for feat in feats] # cl takes one /arch: level and does not define the macros of the extensions above it if "AVX512" in feats: flags, macros = ["/arch:AVX512"], ["GGML_AVX512"] if "AVX512_VBMI" in feats: macros.append("__AVX512VBMI__") if "AVX512_VNNI" in feats: macros += ["__AVX512VNNI__", "GGML_AVX512_VNNI"] elif "AVX2" in feats: flags, macros = ["/arch:AVX2"], ["GGML_AVX2", "GGML_FMA", "GGML_F16C"] elif "AVX" in feats: flags, macros = ["/arch:AVX"], ["GGML_AVX"] elif "SSE42" in feats: flags, macros = ["/arch:SSE4.2"], ["GGML_SSE42"] else: flags, macros = [], [] if "AVX_VNNI" in feats: macros += ["__AVXVNNI__", "GGML_AVX_VNNI"] if "BMI2" in feats: macros += ["__BMI2__", "GGML_BMI2"] return flags, macros def _ggml_cpu_targets() -> tuple[list, list, dict[str, list[str]]]: """(build_clib libraries, variant modules, the static libraries each module links in link order).""" if not GGML_CPU: return [], [], {} if sys.platform == "win32": cflags, c_std, cxx_std, define = ["/O2", "/DNDEBUG", "/utf-8", "/D_CRT_SECURE_NO_WARNINGS"], "/std:c11", "/std:c++20", "/D" # upstream builds these variants only with GCC or Clang variants = {k: v for k, v in GGML_CPU_VARIANTS.items() if k not in ("ivybridge", "piledriver", "cooperlake", "zen4", "sapphirerapids")} else: cflags, c_std, cxx_std, define = ["-O3", "-fPIC", "-fvisibility=hidden", "-DNDEBUG"], "-std=c11", "-std=c++20", "-D" variants = GGML_CPU_VARIANTS build = { "include_dirs": [f"{GGML_DIR}/include", f"{GGML_DIR}/src", f"{GGML_DIR}/ggml-cpu", GGML_DIR], "obj_deps": {"": sorted(str(p.relative_to(ROOT)) for d in ("include", "src", "ggml-cpu", "freetoken") for p in (ROOT / GGML_DIR / d).rglob("*.h"))}, } # ggml-base, built once without ISA flags like upstream's; arch/x86/quants.c calls its quantize_row_*_ref libraries = [("freetoken_ggml_base", {"sources": [f"{GGML_DIR}/src/ggml-quants.c"], "cflags": [*cflags, c_std], **build})] modules, module_libs = [], {} for variant, feats in variants.items(): flags, macros = _ggml_variant_flags(feats) defines = [define + m for m in macros] libraries += [ (f"freetoken_ggml_cpu_{variant}", { "sources": [f"{GGML_DIR}/ggml-cpu/quants.c", f"{GGML_DIR}/ggml-cpu/arch/x86/quants.c", f"{GGML_DIR}/freetoken/cpu_kernels.c"], "cflags": [*cflags, *flags, *defines, c_std], **build, }), # the float row kernels; C++, so a library of their own (f"freetoken_ggml_cpu_vec_{variant}", {"sources": [f"{GGML_DIR}/ggml-cpu/vec.cpp"], "cflags": [*cflags, *flags, *defines, cxx_std], **build}), ] name = f"freetoken.kernel.ggml_cpu._{variant}" module_libs[name] = [f"freetoken_ggml_cpu_{variant}", f"freetoken_ggml_cpu_vec_{variant}", "freetoken_ggml_base"] modules.append(Extension( name=name, sources=[f"{GGML_DIR}/freetoken/variant_module.cpp"], # absolute: ninja compiles from the build directory include_dirs=[str(ROOT / d) for d in (*build["include_dirs"], f"{GGML_DIR}/freetoken")], extra_compile_args=[*cflags, *defines, f"{define}FT_GGML_VARIANT={variant}", cxx_std], )) return libraries, modules, module_libs ggml_cpu_libraries, ggml_cpu_modules, ggml_cpu_module_libs = _ggml_cpu_targets() class BuildClib(build_clib): def build_libraries(self, libraries): # the variants compile the same sources with different flags, so each library gets its own object directory build_temp = self.build_temp try: for lib in libraries: self.build_temp = os.path.join(build_temp, lib[0]) super().build_libraries([lib]) finally: self.build_temp = build_temp class BuildExt(BuildExtension): def run(self): # build_ext alone (`setup.py build_ext --inplace`) does not run build_clib if self.distribution.has_c_libraries(): self.run_command("build_clib") # build_ext would link every build_clib library into every extension; build_extension adds each module's own libraries, self.distribution.libraries = self.distribution.libraries, None try: super().run() finally: self.distribution.libraries = libraries def build_extension(self, ext): if ext.name in ggml_cpu_module_libs: build_clib_dir = self.get_finalized_command("build_clib").build_clib if build_clib_dir not in ext.library_dirs: ext.library_dirs.append(build_clib_dir) for name in ggml_cpu_module_libs[ext.name]: if name not in ext.libraries: ext.libraries.append(name) lib = os.path.join(build_clib_dir, self.compiler.library_filename(name)) # relink when only a static library changed if lib not in ext.depends: ext.depends.append(lib) # every variant module compiles the same source, so each gets its own object directory build_temp, self.build_temp = self.build_temp, os.path.join(self.build_temp, ext.name) try: super().build_extension(ext) finally: self.build_temp = build_temp return super().build_extension(ext) setup( libraries=ggml_cpu_libraries, ext_modules=[ CppExtension( name="freetoken.kernel._pinned_tensor", sources=[ "python/freetoken/kernel/csrc/pinned_tensor.cpp", ], include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs], library_dirs=runtime_library_dirs, libraries=[runtime_lib], extra_compile_args=extra_compile, extra_link_args=runtime_link_args, ), # CPU-compute MoE executor for --moe-backend cpu. Links cudart/amdhip64 for the # cudaLaunchHostFunc submit/sync graph nodes; the bf16 GEMV microkernels # use per-function target attributes (avx512bf16/avx512f) + a runtime # __builtin_cpu_supports dispatch, so the single binary stays portable # (scalar fallback) -- no global -march is set. CppExtension( name="freetoken.kernel._cpu_moe", sources=[ "python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp", "python/freetoken/kernel/csrc/cpu_moe/locked_fetch.cpp", "python/freetoken/kernel/csrc/cpu_moe/locked_fetch_io.cu", f"python/freetoken/kernel/csrc/cpu_moe/gpu_sync_{'hip' if IS_ROCM else 'cuda'}.cpp", "python/freetoken/kernel/csrc/cpu_moe/gpu_sync_io.cu", *sorted( str(p.relative_to(ROOT)) for p in (ROOT / "python/freetoken/kernel/csrc/cpu_moe/kernels").glob("*.cpp") ), ], include_dirs=[KERNEL_INCLUDE, *runtime_include_dirs, str(ROOT / GGML_DIR / "freetoken")], library_dirs=runtime_library_dirs, libraries=[runtime_lib], extra_compile_args={"cxx": extra_compile + ["-pthread"], "nvcc": device_compile}, extra_link_args=runtime_link_args, ), # disk-backed row store (PLE / Engram tables); Linux-only until the TableFile/BatchReader seams grow Windows bodies *([ CppExtension( name="freetoken.kernel._row_store", sources=[ "python/freetoken/kernel/csrc/row_store/row_store_ext.cpp", ], include_dirs=runtime_include_dirs if IS_ROCM else [], library_dirs=runtime_library_dirs if IS_ROCM else [], libraries=[runtime_lib] if IS_ROCM else [], extra_compile_args=["-O3", "-std=c++20"], extra_link_args=runtime_link_args if IS_ROCM else [], ) ] if sys.platform == "linux" else []), *ggml_cpu_modules, ], cmdclass={"build_clib": BuildClib, "build_ext": BuildExt.with_options(use_ninja=True)}, )