mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
297 lines
9.8 KiB
Python
297 lines
9.8 KiB
Python
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import contextlib
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
from collections.abc import Iterable, Iterator
|
|
from dataclasses import dataclass
|
|
from typing import Callable
|
|
|
|
from .aot_models import (
|
|
aggregate_fast_index_copy_feature_sizes,
|
|
aggregate_index_variants,
|
|
aggregate_store_element_sizes,
|
|
)
|
|
from .utils import (
|
|
DISABLE_JIT_ENV,
|
|
DISABLE_KERNEL_CACHE_ENV,
|
|
_make_name,
|
|
load_aot,
|
|
load_jit,
|
|
make_cpp_args,
|
|
)
|
|
|
|
BuildFn = Callable[[pathlib.Path], object]
|
|
|
|
# Shape coverage is generated from the explicit per-model support table in
|
|
# kernel/aot_models.py; add new models/formats there, not here.
|
|
DEFAULT_STORE_ELEMENT_SIZES = aggregate_store_element_sizes()
|
|
DEFAULT_INDEX_VARIANTS = aggregate_index_variants()
|
|
# Closed, model-independent set: max_k = next_pow2(top_k) capped at
|
|
# ENSURE_MAX_K_CAP=16, so these four buckets cover any top_k in [1, 16].
|
|
DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES = aggregate_fast_index_copy_feature_sizes()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class KernelSpec:
|
|
name: str
|
|
build: BuildFn
|
|
|
|
|
|
def _store_spec(element_size: int) -> KernelSpec:
|
|
from .store import DEFAULT_INDEX_KERNEL_CONFIG
|
|
|
|
args = make_cpp_args(element_size, *DEFAULT_INDEX_KERNEL_CONFIG)
|
|
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_jit(
|
|
"store",
|
|
*args,
|
|
cuda_files=["store.cu"],
|
|
cuda_wrappers=[("launch", f"StoreKernel<{args}>::run")],
|
|
build_directory=str(build_directory),
|
|
)
|
|
|
|
return KernelSpec(name=_make_name("store", *args), build=build)
|
|
|
|
|
|
def _index_spec(element_size: int, num_splits: int) -> KernelSpec:
|
|
from .index import DEFAULT_INDEX_KERNEL_CONFIG
|
|
|
|
args = make_cpp_args(element_size, num_splits, *DEFAULT_INDEX_KERNEL_CONFIG)
|
|
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_jit(
|
|
"index",
|
|
*args,
|
|
cuda_files=["index.cu"],
|
|
cuda_wrappers=[("launch", f"IndexKernel<{args}>::run")],
|
|
build_directory=str(build_directory),
|
|
)
|
|
|
|
return KernelSpec(name=_make_name("index", *args), build=build)
|
|
|
|
|
|
|
|
def _fast_index_copy_spec(feature_size: int) -> KernelSpec:
|
|
from .fast_index_copy import default_worker_args
|
|
|
|
worker_threads, worker_feature_size, num_block = default_worker_args(feature_size)
|
|
args = make_cpp_args(feature_size, worker_threads, worker_feature_size, 1024, num_block, 1)
|
|
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_jit(
|
|
"fast_index_copy",
|
|
*args,
|
|
cuda_files=["fast_index_copy.cuh"],
|
|
cuda_wrappers=[
|
|
("launch", f"&FastIndexCopyKernel<{args}>::run"),
|
|
("launch_high", f"&FastIndexCopyKernel<{args}>::run_high"),
|
|
("launch_normal", f"&FastIndexCopyKernel<{args}>::run_normal"),
|
|
],
|
|
build_directory=str(build_directory),
|
|
)
|
|
|
|
return KernelSpec(name=_make_name("fast_index_copy", *args), build=build)
|
|
|
|
|
|
def _fast_index_copy_update_flag_spec() -> KernelSpec:
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_jit(
|
|
"fast_index_copy_update_flag",
|
|
cuda_files=["fast_index_copy.cuh"],
|
|
cuda_wrappers=[("update_copy_flag", "update_copy_flag")],
|
|
build_directory=str(build_directory),
|
|
)
|
|
|
|
return KernelSpec(name=_make_name("fast_index_copy_update_flag"), build=build)
|
|
|
|
|
|
def _fast_index_copy_multi_spec(num_threads: int, blocks_per_bank: int) -> KernelSpec:
|
|
args = make_cpp_args(num_threads, blocks_per_bank)
|
|
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_jit(
|
|
"fast_index_copy_multi",
|
|
*args,
|
|
cuda_files=["fast_index_copy.cuh"],
|
|
cuda_wrappers=[("launch", f"&MultiIndexCopyKernel<{args}>::run")],
|
|
build_directory=str(build_directory),
|
|
)
|
|
|
|
return KernelSpec(name=_make_name("fast_index_copy_multi", *args), build=build)
|
|
|
|
|
|
def _batch_memcpy_spec() -> KernelSpec:
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_jit(
|
|
"batch_memcpy",
|
|
cuda_files=["batch_memcpy.cuh"],
|
|
cuda_wrappers=[("batch_memcpy", "&BatchMemcpy::run")],
|
|
build_directory=str(build_directory),
|
|
)
|
|
|
|
return KernelSpec(name=_make_name("batch_memcpy"), build=build)
|
|
|
|
|
|
def _radix_spec() -> KernelSpec:
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
return load_aot("radix", cpp_files=["radix.cpp"], build_directory=str(build_directory))
|
|
|
|
return KernelSpec(name=_make_name("radix"), build=build)
|
|
|
|
|
|
def _ggml_spec() -> KernelSpec:
|
|
def build(build_directory: pathlib.Path) -> object:
|
|
from .ggml import load_module
|
|
|
|
return load_module(str(build_directory))
|
|
|
|
return KernelSpec(name=_make_name("ggml"), build=build)
|
|
|
|
|
|
def default_kernel_specs() -> tuple[KernelSpec, ...]:
|
|
specs: list[KernelSpec] = []
|
|
specs.extend(_store_spec(element_size) for element_size in DEFAULT_STORE_ELEMENT_SIZES)
|
|
specs.extend(_index_spec(*variant) for variant in DEFAULT_INDEX_VARIANTS)
|
|
specs.append(_fast_index_copy_update_flag_spec())
|
|
specs.extend(
|
|
_fast_index_copy_spec(feature_size)
|
|
for feature_size in DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES
|
|
)
|
|
specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=8))
|
|
specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=32))
|
|
# prefill hit-D2D gather (HBM-bound: wide grid) + its miss-side batch H2D binding.
|
|
specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=64))
|
|
specs.append(_batch_memcpy_spec())
|
|
specs.append(_radix_spec())
|
|
specs.append(_ggml_spec())
|
|
|
|
names = [spec.name for spec in specs]
|
|
duplicates = {name for name in names if names.count(name) > 1}
|
|
if duplicates:
|
|
raise RuntimeError(f"Duplicate kernel cache specs: {sorted(duplicates)}")
|
|
return tuple(specs)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def _force_source_build() -> Iterator[None]:
|
|
old_disable_cache = os.environ.get(DISABLE_KERNEL_CACHE_ENV)
|
|
old_disable_jit = os.environ.get(DISABLE_JIT_ENV)
|
|
os.environ[DISABLE_KERNEL_CACHE_ENV] = "1"
|
|
os.environ.pop(DISABLE_JIT_ENV, None)
|
|
try:
|
|
yield
|
|
finally:
|
|
if old_disable_cache is None:
|
|
os.environ.pop(DISABLE_KERNEL_CACHE_ENV, None)
|
|
else:
|
|
os.environ[DISABLE_KERNEL_CACHE_ENV] = old_disable_cache
|
|
if old_disable_jit is None:
|
|
os.environ.pop(DISABLE_JIT_ENV, None)
|
|
else:
|
|
os.environ[DISABLE_JIT_ENV] = old_disable_jit
|
|
|
|
|
|
def _resolve_specs(specs: Iterable[KernelSpec | str] | None) -> tuple[KernelSpec, ...]:
|
|
default_specs = default_kernel_specs()
|
|
if specs is None:
|
|
return default_specs
|
|
|
|
by_name = {spec.name: spec for spec in default_specs}
|
|
resolved: list[KernelSpec] = []
|
|
for spec in specs:
|
|
if isinstance(spec, KernelSpec):
|
|
resolved.append(spec)
|
|
continue
|
|
try:
|
|
resolved.append(by_name[spec])
|
|
except KeyError as exc:
|
|
raise KeyError(f"Unknown default kernel spec {spec!r}") from exc
|
|
return tuple(resolved)
|
|
|
|
|
|
def _built_shared_object(build_dir: pathlib.Path, name: str) -> pathlib.Path:
|
|
candidates = [
|
|
path
|
|
for path in build_dir.rglob(f"{name}.so")
|
|
if path.parent.name == name or path.parent.name.startswith(f"{name}_")
|
|
]
|
|
if not candidates:
|
|
raise RuntimeError(f"Compiled kernel {name!r} was not found under {build_dir}")
|
|
return max(candidates, key=lambda path: path.stat().st_mtime_ns)
|
|
|
|
|
|
def _copy_kernel(build_dir: pathlib.Path, out_dir: pathlib.Path, name: str) -> pathlib.Path:
|
|
src = _built_shared_object(build_dir, name)
|
|
dst_dir = out_dir / name
|
|
if dst_dir.exists():
|
|
shutil.rmtree(dst_dir)
|
|
dst_dir.mkdir(parents=True, exist_ok=True)
|
|
dst = dst_dir / f"{name}.so"
|
|
shutil.copy2(src, dst)
|
|
return dst
|
|
|
|
|
|
KERNEL_CACHE_JOBS_ENV = "FREETOKEN_KERNEL_CACHE_JOBS"
|
|
|
|
|
|
def _default_jobs(num_specs: int) -> int:
|
|
raw = os.getenv(KERNEL_CACHE_JOBS_ENV, "").strip()
|
|
if raw:
|
|
return max(1, int(raw))
|
|
return max(1, min(num_specs, os.cpu_count() or 1))
|
|
|
|
|
|
def compile_and_package_kernels(
|
|
*,
|
|
out_dir: str | os.PathLike[str],
|
|
build_dir: str | os.PathLike[str],
|
|
specs: Iterable[KernelSpec | str] | None = None,
|
|
clean: bool = True,
|
|
verbose: bool = False,
|
|
jobs: int | None = None,
|
|
) -> list[pathlib.Path]:
|
|
out_path = pathlib.Path(out_dir)
|
|
build_path = pathlib.Path(build_dir)
|
|
selected_specs = _resolve_specs(specs)
|
|
if jobs is None:
|
|
jobs = _default_jobs(len(selected_specs))
|
|
|
|
if clean and out_path.exists():
|
|
shutil.rmtree(out_path)
|
|
out_path.mkdir(parents=True, exist_ok=True)
|
|
build_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _build_one(spec: KernelSpec) -> pathlib.Path:
|
|
spec_build_dir = build_path / spec.name
|
|
if clean and spec_build_dir.exists():
|
|
shutil.rmtree(spec_build_dir)
|
|
spec_build_dir.mkdir(parents=True, exist_ok=True)
|
|
if verbose:
|
|
print(f"Building {spec.name}", flush=True)
|
|
spec.build(spec_build_dir)
|
|
return _copy_kernel(spec_build_dir, out_path, spec.name)
|
|
|
|
# Specs are independent (each compiles in its own build directory), so they can
|
|
# build concurrently; the compilers run as subprocesses, so threads suffice.
|
|
with _force_source_build():
|
|
if jobs <= 1:
|
|
copied = [_build_one(spec) for spec in selected_specs]
|
|
else:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
|
|
copied = list(pool.map(_build_one, selected_specs))
|
|
|
|
if not copied:
|
|
raise RuntimeError("No kernels were packaged")
|
|
return copied
|
|
|
|
|
|
__all__ = [
|
|
"KernelSpec",
|
|
"compile_and_package_kernels",
|
|
"default_kernel_specs",
|
|
]
|