Files
Xiaoze Fanandzihaomu e5d3bd3faf feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding

* build(deps): vendor the gguf-py reader

* feat(quant): add the GGUF quant kind and an embedding quant method

* feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path

* fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA

* perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16

* build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small

* fix(gguf): read F32, F16 and BF16 PLE tables without ggml

* fix(moe): pad the offload slot cache for ggml MMQ tail reads

* fix(gguf): support hip builds and runtime devices

* fix(gguf): copy same-type dequantize and check for missing ggml converters

* fix(gguf): load quantized qwen4exp ple_key and ple_value

* feat(gguf): run float parts of a fused GGUF linear as bf16 segments

* fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights

* fix(moe): zero the prefill staging area after the last prefill layer

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-15 16:33:16 -07:00

271 lines
11 KiB
Python

"""Checkpoint-loading entry points, routed per model architecture.
Each loader resolves the model spec from the checkpoint config and dispatches to the
model module (``models/<name>/weight.py``). Passing ``dummy=True`` replaces the
checkpoint read with randomly filled tensors that keep the loader's exact output
contract (shapes, dtypes, pinning), so everything downstream — repack, offload cache,
kernels — runs unchanged without weights on disk. A model whose banks differ from the
default layout opts out by defining the same-named ``dummy_*`` hook in its weight
module; otherwise the defaults below (built purely from the parsed config) apply.
"""
from __future__ import annotations
import glob
import json
import mmap
import os
import queue
import struct
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Iterator, Tuple
import torch
from freetoken.utils import cached_load_hf_config
from .register import _load_attr, get_model_spec
# safetensors header dtype strings -> torch dtypes (for the parallel reader below)
_ST_DTYPE = {
"F64": torch.float64, "F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16,
"I64": torch.int64, "I32": torch.int32, "I16": torch.int16, "I8": torch.int8,
"U8": torch.uint8, "BOOL": torch.bool,
"F8_E4M3": torch.float8_e4m3fn, "F8_E5M2": torch.float8_e5m2, "F8_E8M0": torch.float8_e8m0fnu,
}
_ODIRECT_BLK = 4096
def _read_shard_odirect_parallel(path: str, workers: int, chunk: int) -> mmap.mmap:
"""Read a whole shard into a page-aligned mmap via CHUNKED multi-threaded O_DIRECT.
Multi-threading one fd scales even for single-shard checkpoints (measured ~7x at 8
threads): the kernel issues the parallel preads at high queue depth. DMA bypasses the
page cache, so there's nothing to drop afterwards."""
size = os.path.getsize(path)
asize = ((size + _ODIRECT_BLK - 1) // _ODIRECT_BLK) * _ODIRECT_BLK
buf = mmap.mmap(-1, asize)
mv = memoryview(buf)
fd = os.open(path, os.O_RDONLY | os.O_DIRECT)
offs = list(range(0, size, chunk))
def rd(o):
want = min(chunk, asize - o)
want = min(want, ((size - o + _ODIRECT_BLK - 1) // _ODIRECT_BLK) * _ODIRECT_BLK)
os.preadv(fd, [mv[o:o + want]], o)
try:
if len(offs) <= 1:
for o in offs:
rd(o)
else:
with ThreadPoolExecutor(workers) as ex:
list(ex.map(rd, offs))
finally:
os.close(fd)
return buf
def iter_expert_tensors_parallel(
model_path: str,
is_expert: Callable[[str], bool],
*,
workers: int = 8,
chunk: int = 8 << 20,
drop_cache: bool = True,
prefetch: int = 2,
) -> Iterator[Tuple[str, torch.Tensor]]:
"""Parallel O_DIRECT analog of a model's serial expert ``iter_weights``.
Common scaffolding for the "parallel" load path: reads every shard that holds >=1 expert
tensor (``is_expert(name)`` is the model's per-model predicate) with chunked
multi-threaded O_DIRECT, parses the safetensors header, and yields ``(name, tensor)``
in checkpoint dtype/shape for the expert tensors.
A background reader PREFETCHES the next ``prefetch`` shards (each chunked O_DIRECT)
while the consumer places the current one, so the disk stays busy during placement
instead of idling between shards (the gap that made the naive sequential version slow).
Peak host memory is ~(prefetch+1) shards + the banks the caller fills. Order is
shard-then-header order (NOT global), so the consumer must place by ``name``.
"""
from freetoken.models.loader import safetensors_weight_map
from freetoken.utils.hf import download_hf_weight
model_path = download_hf_weight(model_path) # resolve hub id -> local (parity w/ serial)
weight_map = safetensors_weight_map(model_path)
shards: dict[str, list[str]] = {}
for name, shard in weight_map.items():
if is_expert(name):
shards.setdefault(shard, []).append(name)
shard_list = sorted(shards)
q: queue.Queue = queue.Queue(maxsize=max(1, prefetch))
_DONE = object()
err: list[BaseException] = []
def _reader():
try:
for shard in shard_list:
path = os.path.join(model_path, shard)
if drop_cache:
try:
fd = os.open(path, os.O_RDONLY)
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
os.close(fd)
except OSError:
pass
buf = _read_shard_odirect_parallel(path, workers, chunk) # overlaps placement
n = struct.unpack("<Q", bytes(buf[:8]))[0]
hdr = json.loads(bytes(buf[8:8 + n]))
q.put((buf, hdr, 8 + n, shards[shard], os.path.getsize(path)))
except BaseException as e: # surface reader errors to the consumer
err.append(e)
finally:
q.put(_DONE)
from freetoken.utils.progress import byte_bar
th = threading.Thread(target=_reader, name="expert-prefetch", daemon=True)
th.start()
bar = byte_bar(sum(os.path.getsize(os.path.join(model_path, s)) for s in shard_list),
"Loading experts (parallel)")
try:
while True:
item = q.get()
if item is _DONE:
break
buf, hdr, base, names, shard_sz = item
mv = memoryview(buf)
for name in names:
meta = hdr[name]
b, e = meta["data_offsets"]
t = torch.frombuffer(mv[base + b: base + e], dtype=_ST_DTYPE[meta["dtype"]])
yield name, (t.view(*meta["shape"]) if meta["shape"] else t)
bar.update(shard_sz)
del mv, buf # freed once the consumer drops the last yielded tensor of this shard
finally:
bar.close()
th.join()
if err:
raise err[0]
_SCATTERED_AVG_BYTES = 16 << 20 # avg expert tensor below this -> "scattered" -> prefer parallel
def experts_scattered(model_path: str) -> bool:
"""Slow-path strategy signal: are the experts stored as many SMALL tensors?
If yes (per-expert / quantized layouts -> avg expert tensor a few MiB), the serial
baseline pays per-tensor overhead on thousands of tiny reads and is slow, so the parallel
parallel whole-shard O_DIRECT reader wins. If experts are pre-packed into a few large
tensors, the serial read already saturates the disk and parallel only adds read amplification.
Measured cheaply from the safetensors headers (no tensor data read). This is a best-
effort heuristic: ANY failure (unresolvable path, no safetensors, GGUF, unreadable
header) -> False (serial), so the real loader still runs and reports real errors."""
try:
from freetoken.utils.hf import download_hf_weight
model_path = download_hf_weight(model_path) # resolve hub ids -> local (parity w/ serial)
index = os.path.join(model_path, "model.safetensors.index.json")
if os.path.exists(index):
with open(index) as f:
shards = sorted(set(json.load(f)["weight_map"].values()))
else:
shards = sorted(os.path.basename(p) for p in glob.glob(os.path.join(model_path, "*.safetensors")))
sizes: list[int] = []
for shard in shards:
try:
with open(os.path.join(model_path, shard), "rb") as fh:
n = struct.unpack("<Q", fh.read(8))[0]
hdr = json.loads(fh.read(n))
except (OSError, ValueError, struct.error): # unreadable/partial shard -> skip
continue
for name, meta in hdr.items():
if name != "__metadata__" and ".experts." in name:
b, e = meta["data_offsets"]
sizes.append(e - b)
if not sizes:
return False
return (sum(sizes) / len(sizes)) < _SCATTERED_AVG_BYTES
except Exception: # heuristic only -> default to serial; the real loader reports errors
return False
def _spec_for_model_path(model_path: str):
hf_config = cached_load_hf_config(model_path)
spec = get_model_spec(hf_config.architectures[0])
parse_config = _load_attr(spec.module, spec.parse_config)
return parse_config(hf_config), spec
def _model_override(spec, name: str):
"""The model module's same-named hook, if it defines one."""
try:
return _load_attr(spec.module, name)
except AttributeError:
return None
def load_weight(
model_path: str,
device: torch.device,
*,
include_moe_experts: bool = True,
include_vision: bool = True,
) -> Iterator[Tuple[str, torch.Tensor]]:
# FTW checkpoint: dense weights are stored post-iter_weights, so we replay them
# model-agnostically instead of re-running the per-model reader. Which tensors exist is
# decided at conversion (offload -> experts live in banks, not here); a backend mismatch
# fails loudly in load_state_dict (strict missing/unexpected expert keys), so the reader
# just yields the stored weight tensors regardless of the include_moe_experts flag.
from freetoken.checkpoint.ftw import is_ftw_checkpoint, iter_ftw_weights
from freetoken.models.config import VISION_KEY_PREFIXES
# a text-only engine never built the tower, so its tensors are not even read
keep = None if include_vision else (lambda name: not name.startswith(VISION_KEY_PREFIXES))
if is_ftw_checkpoint(model_path):
weights = iter_ftw_weights(model_path, keep=keep)
else:
_config, spec = _spec_for_model_path(model_path)
iter_weights = _load_attr(spec.module, spec.iter_weights)
# only a family that registers an encoder is asked about the tower; the others never load one
kwargs = {"include_vision": include_vision} if spec.encoders else {}
weights = iter_weights(
model_path,
device,
include_moe_experts=include_moe_experts,
include_non_moe=True,
**kwargs,
)
for name, tensor in weights:
if keep is not None and not keep(name):
continue
yield name, tensor
def load_vision_weight(model_path: str, device: torch.device) -> Iterator[Tuple[str, torch.Tensor]]:
"""The vision encoder tensors alone, named as load_weight names them, read by the family's encoder-only reader."""
_config, spec = _spec_for_model_path(model_path)
reader = _model_override(spec, "iter_vision_weights")
if reader is None:
raise ValueError(f"{spec.module} has no encoder-only weight reader")
return reader(model_path, device)
def ftw_lacks_vision(model_path: str) -> bool:
"""True for an FTW that holds no vision encoder tensors: converted by a build before the family served images."""
from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
from freetoken.models.config import VISION_KEY_PREFIXES
if not is_ftw_checkpoint(model_path):
return False
return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight"))
__all__ = [
"load_weight",
"load_vision_weight",
"ftw_lacks_vision",
"iter_expert_tensors_parallel",
]