mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
271 lines
11 KiB
Python
271 lines
11 KiB
Python
"""Checkpoint-loading entry points, routed per model architecture.
|
|
|
|
Each loader resolves the model spec from the checkpoint config and dispatches to the
|
|
model module (``models/<name>/weight.py``). Passing ``dummy=True`` replaces the
|
|
checkpoint read with randomly filled tensors that keep the loader's exact output
|
|
contract (shapes, dtypes, pinning), so everything downstream — repack, offload cache,
|
|
kernels — runs unchanged without weights on disk. A model whose banks differ from the
|
|
default layout opts out by defining the same-named ``dummy_*`` hook in its weight
|
|
module; otherwise the defaults below (built purely from the parsed config) apply.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import json
|
|
import mmap
|
|
import os
|
|
import queue
|
|
import struct
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Callable, Iterator, Tuple
|
|
|
|
import torch
|
|
from freetoken.utils import cached_load_hf_config
|
|
|
|
from .register import _load_attr, get_model_spec
|
|
|
|
# safetensors header dtype strings -> torch dtypes (for the parallel reader below)
|
|
_ST_DTYPE = {
|
|
"F64": torch.float64, "F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16,
|
|
"I64": torch.int64, "I32": torch.int32, "I16": torch.int16, "I8": torch.int8,
|
|
"U8": torch.uint8, "BOOL": torch.bool,
|
|
"F8_E4M3": torch.float8_e4m3fn, "F8_E5M2": torch.float8_e5m2, "F8_E8M0": torch.float8_e8m0fnu,
|
|
}
|
|
_ODIRECT_BLK = 4096
|
|
|
|
|
|
def _read_shard_odirect_parallel(path: str, workers: int, chunk: int) -> mmap.mmap:
|
|
"""Read a whole shard into a page-aligned mmap via CHUNKED multi-threaded O_DIRECT.
|
|
Multi-threading one fd scales even for single-shard checkpoints (measured ~7x at 8
|
|
threads): the kernel issues the parallel preads at high queue depth. DMA bypasses the
|
|
page cache, so there's nothing to drop afterwards."""
|
|
size = os.path.getsize(path)
|
|
asize = ((size + _ODIRECT_BLK - 1) // _ODIRECT_BLK) * _ODIRECT_BLK
|
|
buf = mmap.mmap(-1, asize)
|
|
mv = memoryview(buf)
|
|
fd = os.open(path, os.O_RDONLY | os.O_DIRECT)
|
|
offs = list(range(0, size, chunk))
|
|
|
|
def rd(o):
|
|
want = min(chunk, asize - o)
|
|
want = min(want, ((size - o + _ODIRECT_BLK - 1) // _ODIRECT_BLK) * _ODIRECT_BLK)
|
|
os.preadv(fd, [mv[o:o + want]], o)
|
|
|
|
try:
|
|
if len(offs) <= 1:
|
|
for o in offs:
|
|
rd(o)
|
|
else:
|
|
with ThreadPoolExecutor(workers) as ex:
|
|
list(ex.map(rd, offs))
|
|
finally:
|
|
os.close(fd)
|
|
return buf
|
|
|
|
|
|
def iter_expert_tensors_parallel(
|
|
model_path: str,
|
|
is_expert: Callable[[str], bool],
|
|
*,
|
|
workers: int = 8,
|
|
chunk: int = 8 << 20,
|
|
drop_cache: bool = True,
|
|
prefetch: int = 2,
|
|
) -> Iterator[Tuple[str, torch.Tensor]]:
|
|
"""Parallel O_DIRECT analog of a model's serial expert ``iter_weights``.
|
|
|
|
Common scaffolding for the "parallel" load path: reads every shard that holds >=1 expert
|
|
tensor (``is_expert(name)`` is the model's per-model predicate) with chunked
|
|
multi-threaded O_DIRECT, parses the safetensors header, and yields ``(name, tensor)``
|
|
in checkpoint dtype/shape for the expert tensors.
|
|
|
|
A background reader PREFETCHES the next ``prefetch`` shards (each chunked O_DIRECT)
|
|
while the consumer places the current one, so the disk stays busy during placement
|
|
instead of idling between shards (the gap that made the naive sequential version slow).
|
|
Peak host memory is ~(prefetch+1) shards + the banks the caller fills. Order is
|
|
shard-then-header order (NOT global), so the consumer must place by ``name``.
|
|
"""
|
|
from freetoken.models.loader import safetensors_weight_map
|
|
from freetoken.utils.hf import download_hf_weight
|
|
|
|
model_path = download_hf_weight(model_path) # resolve hub id -> local (parity w/ serial)
|
|
weight_map = safetensors_weight_map(model_path)
|
|
shards: dict[str, list[str]] = {}
|
|
for name, shard in weight_map.items():
|
|
if is_expert(name):
|
|
shards.setdefault(shard, []).append(name)
|
|
shard_list = sorted(shards)
|
|
|
|
q: queue.Queue = queue.Queue(maxsize=max(1, prefetch))
|
|
_DONE = object()
|
|
err: list[BaseException] = []
|
|
|
|
def _reader():
|
|
try:
|
|
for shard in shard_list:
|
|
path = os.path.join(model_path, shard)
|
|
if drop_cache:
|
|
try:
|
|
fd = os.open(path, os.O_RDONLY)
|
|
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
|
|
os.close(fd)
|
|
except OSError:
|
|
pass
|
|
buf = _read_shard_odirect_parallel(path, workers, chunk) # overlaps placement
|
|
n = struct.unpack("<Q", bytes(buf[:8]))[0]
|
|
hdr = json.loads(bytes(buf[8:8 + n]))
|
|
q.put((buf, hdr, 8 + n, shards[shard], os.path.getsize(path)))
|
|
except BaseException as e: # surface reader errors to the consumer
|
|
err.append(e)
|
|
finally:
|
|
q.put(_DONE)
|
|
|
|
from freetoken.utils.progress import byte_bar
|
|
|
|
th = threading.Thread(target=_reader, name="expert-prefetch", daemon=True)
|
|
th.start()
|
|
bar = byte_bar(sum(os.path.getsize(os.path.join(model_path, s)) for s in shard_list),
|
|
"Loading experts (parallel)")
|
|
try:
|
|
while True:
|
|
item = q.get()
|
|
if item is _DONE:
|
|
break
|
|
buf, hdr, base, names, shard_sz = item
|
|
mv = memoryview(buf)
|
|
for name in names:
|
|
meta = hdr[name]
|
|
b, e = meta["data_offsets"]
|
|
t = torch.frombuffer(mv[base + b: base + e], dtype=_ST_DTYPE[meta["dtype"]])
|
|
yield name, (t.view(*meta["shape"]) if meta["shape"] else t)
|
|
bar.update(shard_sz)
|
|
del mv, buf # freed once the consumer drops the last yielded tensor of this shard
|
|
finally:
|
|
bar.close()
|
|
th.join()
|
|
if err:
|
|
raise err[0]
|
|
|
|
|
|
_SCATTERED_AVG_BYTES = 16 << 20 # avg expert tensor below this -> "scattered" -> prefer parallel
|
|
|
|
|
|
def experts_scattered(model_path: str) -> bool:
|
|
"""Slow-path strategy signal: are the experts stored as many SMALL tensors?
|
|
|
|
If yes (per-expert / quantized layouts -> avg expert tensor a few MiB), the serial
|
|
baseline pays per-tensor overhead on thousands of tiny reads and is slow, so the parallel
|
|
parallel whole-shard O_DIRECT reader wins. If experts are pre-packed into a few large
|
|
tensors, the serial read already saturates the disk and parallel only adds read amplification.
|
|
Measured cheaply from the safetensors headers (no tensor data read). This is a best-
|
|
effort heuristic: ANY failure (unresolvable path, no safetensors, GGUF, unreadable
|
|
header) -> False (serial), so the real loader still runs and reports real errors."""
|
|
try:
|
|
from freetoken.utils.hf import download_hf_weight
|
|
|
|
model_path = download_hf_weight(model_path) # resolve hub ids -> local (parity w/ serial)
|
|
index = os.path.join(model_path, "model.safetensors.index.json")
|
|
if os.path.exists(index):
|
|
with open(index) as f:
|
|
shards = sorted(set(json.load(f)["weight_map"].values()))
|
|
else:
|
|
shards = sorted(os.path.basename(p) for p in glob.glob(os.path.join(model_path, "*.safetensors")))
|
|
sizes: list[int] = []
|
|
for shard in shards:
|
|
try:
|
|
with open(os.path.join(model_path, shard), "rb") as fh:
|
|
n = struct.unpack("<Q", fh.read(8))[0]
|
|
hdr = json.loads(fh.read(n))
|
|
except (OSError, ValueError, struct.error): # unreadable/partial shard -> skip
|
|
continue
|
|
for name, meta in hdr.items():
|
|
if name != "__metadata__" and ".experts." in name:
|
|
b, e = meta["data_offsets"]
|
|
sizes.append(e - b)
|
|
if not sizes:
|
|
return False
|
|
return (sum(sizes) / len(sizes)) < _SCATTERED_AVG_BYTES
|
|
except Exception: # heuristic only -> default to serial; the real loader reports errors
|
|
return False
|
|
|
|
|
|
def _spec_for_model_path(model_path: str):
|
|
hf_config = cached_load_hf_config(model_path)
|
|
spec = get_model_spec(hf_config.architectures[0])
|
|
parse_config = _load_attr(spec.module, spec.parse_config)
|
|
return parse_config(hf_config), spec
|
|
|
|
|
|
def _model_override(spec, name: str):
|
|
"""The model module's same-named hook, if it defines one."""
|
|
try:
|
|
return _load_attr(spec.module, name)
|
|
except AttributeError:
|
|
return None
|
|
|
|
|
|
def load_weight(
|
|
model_path: str,
|
|
device: torch.device,
|
|
*,
|
|
include_moe_experts: bool = True,
|
|
include_vision: bool = True,
|
|
) -> Iterator[Tuple[str, torch.Tensor]]:
|
|
# FTW checkpoint: dense weights are stored post-iter_weights, so we replay them
|
|
# model-agnostically instead of re-running the per-model reader. Which tensors exist is
|
|
# decided at conversion (offload -> experts live in banks, not here); a backend mismatch
|
|
# fails loudly in load_state_dict (strict missing/unexpected expert keys), so the reader
|
|
# just yields the stored weight tensors regardless of the include_moe_experts flag.
|
|
from freetoken.checkpoint.ftw import is_ftw_checkpoint, iter_ftw_weights
|
|
from freetoken.models.config import VISION_KEY_PREFIXES
|
|
|
|
# a text-only engine never built the tower, so its tensors are not even read
|
|
keep = None if include_vision else (lambda name: not name.startswith(VISION_KEY_PREFIXES))
|
|
if is_ftw_checkpoint(model_path):
|
|
weights = iter_ftw_weights(model_path, keep=keep)
|
|
else:
|
|
_config, spec = _spec_for_model_path(model_path)
|
|
iter_weights = _load_attr(spec.module, spec.iter_weights)
|
|
# only a family that registers an encoder is asked about the tower; the others never load one
|
|
kwargs = {"include_vision": include_vision} if spec.encoders else {}
|
|
weights = iter_weights(
|
|
model_path,
|
|
device,
|
|
include_moe_experts=include_moe_experts,
|
|
include_non_moe=True,
|
|
**kwargs,
|
|
)
|
|
for name, tensor in weights:
|
|
if keep is not None and not keep(name):
|
|
continue
|
|
yield name, tensor
|
|
|
|
|
|
def load_vision_weight(model_path: str, device: torch.device) -> Iterator[Tuple[str, torch.Tensor]]:
|
|
"""The vision encoder tensors alone, named as load_weight names them, read by the family's encoder-only reader."""
|
|
_config, spec = _spec_for_model_path(model_path)
|
|
reader = _model_override(spec, "iter_vision_weights")
|
|
if reader is None:
|
|
raise ValueError(f"{spec.module} has no encoder-only weight reader")
|
|
return reader(model_path, device)
|
|
|
|
|
|
def ftw_lacks_vision(model_path: str) -> bool:
|
|
"""True for an FTW that holds no vision encoder tensors: converted by a build before the family served images."""
|
|
from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
|
|
from freetoken.models.config import VISION_KEY_PREFIXES
|
|
|
|
if not is_ftw_checkpoint(model_path):
|
|
return False
|
|
return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight"))
|
|
|
|
|
|
__all__ = [
|
|
"load_weight",
|
|
"load_vision_weight",
|
|
"ftw_lacks_vision",
|
|
"iter_expert_tensors_parallel",
|
|
]
|