Files
FreeToken/python/freetoken/models/loader.py
T
2026-08-11 22:53:25 +00:00

322 lines
11 KiB
Python

from __future__ import annotations
import glob
import json
import os
import re
from dataclasses import dataclass
from typing import Iterable, Iterator
import torch
from freetoken.utils import div_ceil, download_hf_weight
SPLIT_DIM_0 = (".q_proj", ".k_proj", ".v_proj", ".gate_proj", ".up_proj")
SPLIT_DIM_1 = (".o_proj", ".down_proj")
@dataclass(frozen=True)
class MergeRule:
fused_suffix: str
slot: str
slots: tuple[str, ...]
def shard_tensor(
key: str,
value: torch.Tensor,
*,
rank: int,
world_size: int,
num_kv_heads: int,
) -> torch.Tensor:
if any(key.count(sub) for sub in SPLIT_DIM_0):
is_kv_proj = any(key.count(sub) for sub in (".k_proj", ".v_proj"))
if is_kv_proj and num_kv_heads is not None and num_kv_heads < world_size:
head_dim = value.shape[0] // num_kv_heads
head_idx = rank * num_kv_heads // world_size
return value[head_idx * head_dim : (head_idx + 1) * head_dim].clone()
return value.chunk(world_size, dim=0)[rank].clone()
if any(key.count(sub) for sub in SPLIT_DIM_1):
return value.chunk(world_size, dim=1)[rank].clone()
if key.count("lm_head") or key.count("embed_tokens"):
num_embeddings = value.shape[0]
num_embeddings_per_partition = div_ceil(num_embeddings, world_size)
vocab_start_idx = rank * num_embeddings_per_partition
vocab_end_idx = min((rank + 1) * num_embeddings_per_partition, num_embeddings)
return value[vocab_start_idx:vocab_end_idx, :].clone()
return value
def iter_weight_files(model_path: str) -> list[str]:
model_folder = download_hf_weight(model_path)
files = glob.glob(f"{model_folder}/*.safetensors")
return [f for f in files if not f.endswith("consolidated.safetensors")] or files
def drop_page_cache(path: str) -> None:
"""drop a file's page cache: banks + full checkpoint cache don't both fit in host RAM (OOM)."""
try:
fd = os.open(path, os.O_RDONLY)
try:
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
finally:
os.close(fd)
except OSError:
pass
def iter_root_safetensor_files_from_index(
model_path: str,
*,
index_file: str = "model.safetensors.index.json",
) -> list[str]:
model_folder = download_hf_weight(model_path)
if os.path.basename(os.path.normpath(model_folder)) in {"metal", "original"}:
raise ValueError("GPT-OSS loading requires the root GPT-OSS model directory")
root_files = sorted(glob.glob(os.path.join(model_folder, "*.safetensors")))
index_path = os.path.join(model_folder, index_file)
if not os.path.isfile(index_path):
files = [f for f in root_files if not f.endswith("consolidated.safetensors")]
if not files:
raise ValueError("No root GPT-OSS safetensors shards found")
return files
with open(index_path, encoding="utf-8") as f:
weight_map = json.load(f)["weight_map"]
indexed_files = []
for filename in dict.fromkeys(weight_map.values()):
if os.path.dirname(filename):
continue
path = os.path.join(model_folder, filename)
if path in root_files:
indexed_files.append(path)
if not indexed_files:
raise ValueError(
"No root GPT-OSS safetensors shards found from model.safetensors.index.json"
)
return sorted(indexed_files)
def _merge_info(key: str, rules: dict[str, MergeRule]) -> tuple[str, MergeRule] | None:
for suffix, rule in rules.items():
if key.endswith(suffix + ".weight") or key.endswith(suffix) or key.count(suffix):
return key.replace(suffix, rule.fused_suffix), rule
return None
def iter_merged_tensors(
tensors: Iterable[tuple[str, torch.Tensor]],
rules: dict[str, MergeRule],
*,
model_name: str,
) -> Iterator[tuple[str, torch.Tensor]]:
merge_buf: dict[str, dict[str, torch.Tensor]] = {}
for name, tensor in tensors:
info = _merge_info(name, rules)
if info is None:
yield name, tensor
continue
merged_key, rule = info
slots = merge_buf.setdefault(merged_key, {})
slots[rule.slot] = tensor
if not all(slot in slots for slot in rule.slots):
continue
parts = [slots[slot] for slot in rule.slots]
del merge_buf[merged_key]
yield merged_key, torch.cat(parts, dim=0)
assert not merge_buf, (
f"{model_name}: Incomplete merge groups in checkpoint: {list(merge_buf.keys())}"
)
def _expert_stack_info(key: str, expert_pattern: re.Pattern[str]) -> tuple[str, int] | None:
match = expert_pattern.match(key)
if match is None:
return None
packed_name = match.group("name")
if packed_name.endswith(".weight"):
packed_name = packed_name.removesuffix(".weight")
return f"{match.group('prefix')}.{packed_name}", int(match.group("idx"))
def iter_stacked_experts(
tensors: Iterable[tuple[str, torch.Tensor]],
*,
num_experts: int,
model_name: str,
expert_pattern: re.Pattern[str],
) -> Iterator[tuple[str, torch.Tensor]]:
expert_buf: dict[str, dict[int, torch.Tensor]] = {}
for name, tensor in tensors:
expert_info = _expert_stack_info(name, expert_pattern)
if expert_info is None:
yield name, tensor
continue
packed_key, expert_idx = expert_info
slots = expert_buf.setdefault(packed_key, {})
slots[expert_idx] = tensor
if len(slots) != num_experts:
continue
experts = [slots[idx] for idx in range(num_experts)]
del expert_buf[packed_key]
yield packed_key, torch.stack(experts, dim=0)
assert not expert_buf, (
f"{model_name}: Incomplete expert tensors in checkpoint: {list(expert_buf.keys())}"
)
def _packed_expert_source_info(key: str) -> tuple[int, str] | None:
parts = key.split(".")
if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":
return None
if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:
return None
try:
return int(parts[2]), parts[-1]
except ValueError:
return None
class _PlainBank:
"""CPU-only fallback bank: a plain unpinned tensor with a no-op pin (no CUDA)."""
__slots__ = ("tensor",)
def __init__(self, tensor: torch.Tensor):
self.tensor = tensor
def pin(self) -> None:
pass
def _alloc_expert_bank(shape: tuple[int, ...], *, dtype: torch.dtype):
"""Allocate an UNPINNED bank (lazy host mmap), to be filled then pinned at the end --
pin-after-fill. Registering already-resident pages skips cudaHostAlloc's slow commit
(~2.8 GiB/s) / zero-fill. Returns a bank object exposing ``.tensor`` and ``.pin()``."""
if torch.cuda.is_available():
from freetoken.moe.host_banks import HostBank
return HostBank(tuple(shape), dtype)
return _PlainBank(torch.empty(shape, dtype=dtype))
def _copy_expert_layer_into_bank(
banks: dict[str, list],
row_shape: dict[str, tuple[int, ...]],
seen_layers: dict[str, set[int]],
*,
bank_name: str,
tensor: torch.Tensor,
layer: int,
config,
dtype: torch.dtype,
) -> None:
if layer < 0 or layer >= config.num_layers:
raise ValueError(
f"Unexpected MoE expert layer {layer}; expected [0, {config.num_layers})"
)
if tensor.size(0) != config.num_experts:
raise ValueError(
f"Unexpected {bank_name} expert count {tensor.size(0)}; "
f"expected {config.num_experts}"
)
expected_shape = row_shape.setdefault(bank_name, tuple(tensor.shape[1:]))
if tuple(tensor.shape[1:]) != expected_shape:
raise ValueError(
f"Inconsistent {bank_name} expert shape {tuple(tensor.shape[1:])}; "
f"expected {expected_shape}"
)
bank = banks[bank_name][layer]
if bank is None:
banks[bank_name][layer] = bank = _alloc_expert_bank(
(config.num_experts, *tensor.shape[1:]), dtype=dtype
)
bank.tensor.copy_(tensor) # whole-layer arrival; pinned later, after fully resident
seen_layers[bank_name].add(layer)
def stream_moe_expert_sources(
tensors: Iterable[tuple[str, torch.Tensor]],
config,
*,
dtype: torch.dtype,
layer_sink=None,
) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
"""Stream packed per-layer BF16 expert tensors into final offload banks.
Model adapters normalize ordinary MoE expert weights to
``...experts.gate_up_proj`` and ``...experts.down_proj`` with shape
``[num_experts, ...]``. Each arrives whole-layer, so it's written directly into
its own ``[num_experts, ...]`` per-layer bank (independent allocation).
``layer_sink=None`` (serving): pin each layer's banks as its writes complete,
via an internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter):
the tracker fires into it instead -- nothing is pinned, and the sink may release
banks it has written out, so the returned tensors are only valid until then (the
caller owns that tradeoff).
"""
from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
banks: dict[str, list] = { # name -> per-layer [bank obj (HostBank/_PlainBank) or None]
"gate_up": [None] * config.num_layers,
"down": [None] * config.num_layers,
}
row_shape: dict[str, tuple[int, ...]] = {}
seen_layers: dict[str, set[int]] = {"gate_up": set(), "down": set()}
def _load(sink) -> None:
tracker = LayerCompletionTracker(2, banks, sink) # gate_up + down per layer
for name, tensor in tensors:
expert_info = _packed_expert_source_info(name)
if expert_info is None:
raise ValueError(f"Unexpected expert weight key: {name}")
layer, packed_name = expert_info
bank_name = "gate_up" if packed_name == "gate_up_proj" else "down"
_copy_expert_layer_into_bank(
banks,
row_shape,
seen_layers,
bank_name=bank_name,
tensor=tensor,
layer=layer,
config=config,
dtype=dtype,
)
tracker.note(layer)
expected_layers = set(range(config.num_layers))
missing = {
name: sorted(expected_layers - seen)
for name, seen in seen_layers.items()
if seen != expected_layers
}
if missing:
raise ValueError(f"Missing MoE expert source layers: {missing}")
if layer_sink is not None:
_load(layer_sink)
else:
with PinPipeline() as pins:
_load(pins)
return (
[bank.tensor for bank in banks["gate_up"]],
[bank.tensor for bank in banks["down"]],
)
__all__ = [
"MergeRule",
"iter_root_safetensor_files_from_index",
"iter_weight_files",
"iter_merged_tensors",
"iter_stacked_experts",
"shard_tensor",
"stream_moe_expert_sources",
]