"""Checkpoint-loading entry points, routed per model architecture. Each loader resolves the model spec from the checkpoint config and dispatches to the model module (``models//weight.py``). Passing ``dummy=True`` replaces the checkpoint read with randomly filled tensors that keep the loader's exact output contract (shapes, dtypes, pinning), so everything downstream — repack, offload cache, kernels — runs unchanged without weights on disk. A model whose banks differ from the default layout opts out by defining the same-named ``dummy_*`` hook in its weight module; otherwise the defaults below (built purely from the parsed config) apply. """ from __future__ import annotations import glob import json import mmap import os import queue import struct import threading from concurrent.futures import ThreadPoolExecutor from typing import Callable, Iterator, Tuple import torch from freetoken.utils import cached_load_hf_config from .register import _load_attr, get_model_spec # safetensors header dtype strings -> torch dtypes (for the parallel reader below) _ST_DTYPE = { "F64": torch.float64, "F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16, "I64": torch.int64, "I32": torch.int32, "I16": torch.int16, "I8": torch.int8, "U8": torch.uint8, "BOOL": torch.bool, "F8_E4M3": torch.float8_e4m3fn, "F8_E5M2": torch.float8_e5m2, "F8_E8M0": torch.float8_e8m0fnu, } _ODIRECT_BLK = 4096 def _read_shard_odirect_parallel(path: str, workers: int, chunk: int) -> mmap.mmap: """Read a whole shard into a page-aligned mmap via CHUNKED multi-threaded O_DIRECT. Multi-threading one fd scales even for single-shard checkpoints (measured ~7x at 8 threads): the kernel issues the parallel preads at high queue depth. DMA bypasses the page cache, so there's nothing to drop afterwards.""" size = os.path.getsize(path) asize = ((size + _ODIRECT_BLK - 1) // _ODIRECT_BLK) * _ODIRECT_BLK buf = mmap.mmap(-1, asize) mv = memoryview(buf) fd = os.open(path, os.O_RDONLY | os.O_DIRECT) offs = list(range(0, size, chunk)) def rd(o): want = min(chunk, asize - o) want = min(want, ((size - o + _ODIRECT_BLK - 1) // _ODIRECT_BLK) * _ODIRECT_BLK) os.preadv(fd, [mv[o:o + want]], o) try: if len(offs) <= 1: for o in offs: rd(o) else: with ThreadPoolExecutor(workers) as ex: list(ex.map(rd, offs)) finally: os.close(fd) return buf def iter_expert_tensors_parallel( model_path: str, is_expert: Callable[[str], bool], *, workers: int = 8, chunk: int = 8 << 20, drop_cache: bool = True, prefetch: int = 2, ) -> Iterator[Tuple[str, torch.Tensor]]: """Parallel O_DIRECT analog of a model's serial expert ``iter_weights``. Common scaffolding for the "parallel" load path: reads every shard that holds >=1 expert tensor (``is_expert(name)`` is the model's per-model predicate) with chunked multi-threaded O_DIRECT, parses the safetensors header, and yields ``(name, tensor)`` in checkpoint dtype/shape for the expert tensors. A background reader PREFETCHES the next ``prefetch`` shards (each chunked O_DIRECT) while the consumer places the current one, so the disk stays busy during placement instead of idling between shards (the gap that made the naive sequential version slow). Peak host memory is ~(prefetch+1) shards + the banks the caller fills. Order is shard-then-header order (NOT global), so the consumer must place by ``name``. """ from freetoken.models.loader import safetensors_weight_map from freetoken.utils.hf import download_hf_weight model_path = download_hf_weight(model_path) # resolve hub id -> local (parity w/ serial) weight_map = safetensors_weight_map(model_path) shards: dict[str, list[str]] = {} for name, shard in weight_map.items(): if is_expert(name): shards.setdefault(shard, []).append(name) shard_list = sorted(shards) q: queue.Queue = queue.Queue(maxsize=max(1, prefetch)) _DONE = object() err: list[BaseException] = [] def _reader(): try: for shard in shard_list: path = os.path.join(model_path, shard) if drop_cache: try: fd = os.open(path, os.O_RDONLY) os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) os.close(fd) except OSError: pass buf = _read_shard_odirect_parallel(path, workers, chunk) # overlaps placement n = struct.unpack(" "scattered" -> prefer parallel def experts_scattered(model_path: str) -> bool: """Slow-path strategy signal: are the experts stored as many SMALL tensors? If yes (per-expert / quantized layouts -> avg expert tensor a few MiB), the serial baseline pays per-tensor overhead on thousands of tiny reads and is slow, so the parallel parallel whole-shard O_DIRECT reader wins. If experts are pre-packed into a few large tensors, the serial read already saturates the disk and parallel only adds read amplification. Measured cheaply from the safetensors headers (no tensor data read). This is a best- effort heuristic: ANY failure (unresolvable path, no safetensors, GGUF, unreadable header) -> False (serial), so the real loader still runs and reports real errors.""" try: from freetoken.utils.hf import download_hf_weight model_path = download_hf_weight(model_path) # resolve hub ids -> local (parity w/ serial) index = os.path.join(model_path, "model.safetensors.index.json") if os.path.exists(index): with open(index) as f: shards = sorted(set(json.load(f)["weight_map"].values())) else: shards = sorted(os.path.basename(p) for p in glob.glob(os.path.join(model_path, "*.safetensors"))) sizes: list[int] = [] for shard in shards: try: with open(os.path.join(model_path, shard), "rb") as fh: n = struct.unpack(" skip continue for name, meta in hdr.items(): if name != "__metadata__" and ".experts." in name: b, e = meta["data_offsets"] sizes.append(e - b) if not sizes: return False return (sum(sizes) / len(sizes)) < _SCATTERED_AVG_BYTES except Exception: # heuristic only -> default to serial; the real loader reports errors return False def _spec_for_model_path(model_path: str): hf_config = cached_load_hf_config(model_path) spec = get_model_spec(hf_config.architectures[0]) parse_config = _load_attr(spec.module, spec.parse_config) return parse_config(hf_config), spec def _model_override(spec, name: str): """The model module's same-named hook, if it defines one.""" try: return _load_attr(spec.module, name) except AttributeError: return None def load_weight( model_path: str, device: torch.device, *, include_moe_experts: bool = True, include_vision: bool = True, ) -> Iterator[Tuple[str, torch.Tensor]]: # FTW checkpoint: dense weights are stored post-iter_weights, so we replay them # model-agnostically instead of re-running the per-model reader. Which tensors exist is # decided at conversion (offload -> experts live in banks, not here); a backend mismatch # fails loudly in load_state_dict (strict missing/unexpected expert keys), so the reader # just yields the stored weight tensors regardless of the include_moe_experts flag. from freetoken.checkpoint.ftw import is_ftw_checkpoint, iter_ftw_weights from freetoken.models.config import VISION_KEY_PREFIXES # a text-only engine never built the tower, so its tensors are not even read keep = None if include_vision else (lambda name: not name.startswith(VISION_KEY_PREFIXES)) if is_ftw_checkpoint(model_path): weights = iter_ftw_weights(model_path, keep=keep) else: _config, spec = _spec_for_model_path(model_path) iter_weights = _load_attr(spec.module, spec.iter_weights) # only a family that registers an encoder is asked about the tower; the others never load one kwargs = {"include_vision": include_vision} if spec.encoders else {} weights = iter_weights( model_path, device, include_moe_experts=include_moe_experts, include_non_moe=True, **kwargs, ) for name, tensor in weights: if keep is not None and not keep(name): continue yield name, tensor def load_vision_weight(model_path: str, device: torch.device) -> Iterator[Tuple[str, torch.Tensor]]: """The vision encoder tensors alone, named as load_weight names them, read by the family's encoder-only reader.""" _config, spec = _spec_for_model_path(model_path) reader = _model_override(spec, "iter_vision_weights") if reader is None: raise ValueError(f"{spec.module} has no encoder-only weight reader") return reader(model_path, device) def ftw_lacks_vision(model_path: str) -> bool: """True for an FTW that holds no vision encoder tensors: converted by a build before the family served images.""" from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint from freetoken.models.config import VISION_KEY_PREFIXES if not is_ftw_checkpoint(model_path): return False return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight")) __all__ = [ "load_weight", "load_vision_weight", "ftw_lacks_vision", "iter_expert_tensors_parallel", ]