mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* feat(moe): serve experts of several formats through per-format slot pools * refactor(cpu-moe): pass each layer's cpu format with its options * fix(moe): read only each format's layers when loading experts of several formats --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
354 lines
12 KiB
Python
354 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Iterator
|
|
|
|
import safetensors
|
|
import torch
|
|
from freetoken.layers.quantization import QuantKind
|
|
from freetoken.distributed import get_tp_info
|
|
from freetoken.models.loader import (
|
|
MergeRule,
|
|
iter_merged_tensors,
|
|
iter_root_safetensor_files_from_index,
|
|
shard_tensor,
|
|
)
|
|
from freetoken.utils import cached_load_hf_config
|
|
|
|
from .config import parse_config
|
|
|
|
_MERGE_RULES = {
|
|
".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
|
|
".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
|
|
".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
|
|
}
|
|
|
|
|
|
def local_mxfp4_intermediate_range(
|
|
intermediate_size: int,
|
|
*,
|
|
rank: int,
|
|
world_size: int,
|
|
) -> tuple[int, int, int]:
|
|
if intermediate_size % 32 != 0:
|
|
raise ValueError("GPT-OSS MXFP4 intermediate size must be divisible by 32")
|
|
blocks = intermediate_size // 32
|
|
blocks_per_rank = (blocks + world_size - 1) // world_size
|
|
local_intermediate = blocks_per_rank * 32
|
|
start = rank * local_intermediate
|
|
end = min((rank + 1) * local_intermediate, intermediate_size)
|
|
return start, end, local_intermediate
|
|
|
|
|
|
def _shard_dim(value: torch.Tensor, dim: int, start: int, end: int) -> torch.Tensor:
|
|
slices = [slice(None)] * value.ndim
|
|
slices[dim] = slice(start, end)
|
|
return value[tuple(slices)].clone()
|
|
|
|
|
|
def _shard_dim_pad(
|
|
value: torch.Tensor,
|
|
dim: int,
|
|
start: int,
|
|
end: int,
|
|
target_size: int,
|
|
) -> torch.Tensor:
|
|
shard = _shard_dim(value, dim, start, end)
|
|
pad_size = target_size - shard.shape[dim]
|
|
if pad_size <= 0:
|
|
return shard
|
|
pad_shape = list(shard.shape)
|
|
pad_shape[dim] = pad_size
|
|
return torch.cat([shard, shard.new_zeros(pad_shape)], dim=dim)
|
|
|
|
|
|
def _attn_range(
|
|
*,
|
|
kind: str,
|
|
rank: int,
|
|
world_size: int,
|
|
num_q_heads: int,
|
|
num_kv_heads: int,
|
|
head_dim: int,
|
|
) -> tuple[int, int]:
|
|
if kind == "q":
|
|
if num_q_heads % world_size != 0:
|
|
raise ValueError("GPT-OSS query heads must be divisible by TP size")
|
|
heads_per_rank = num_q_heads // world_size
|
|
head_idx = rank * heads_per_rank
|
|
else:
|
|
if world_size > num_kv_heads:
|
|
if world_size % num_kv_heads != 0:
|
|
raise ValueError("GPT-OSS TP size must divide or replicate KV heads")
|
|
heads_per_rank = 1
|
|
head_idx = rank * num_kv_heads // world_size
|
|
else:
|
|
if num_kv_heads % world_size != 0:
|
|
raise ValueError("GPT-OSS KV heads must be divisible by TP size")
|
|
heads_per_rank = num_kv_heads // world_size
|
|
head_idx = rank * heads_per_rank
|
|
return head_idx * head_dim, (head_idx + heads_per_rank) * head_dim
|
|
|
|
|
|
def shard_gpt_oss_tensor(
|
|
name: str,
|
|
value: torch.Tensor,
|
|
*,
|
|
rank: int,
|
|
world_size: int,
|
|
num_q_heads: int,
|
|
num_kv_heads: int,
|
|
head_dim: int,
|
|
intermediate_size: int,
|
|
) -> torch.Tensor:
|
|
if name.endswith(".self_attn.sinks"):
|
|
start, end = _attn_range(
|
|
kind="q",
|
|
rank=rank,
|
|
world_size=world_size,
|
|
num_q_heads=num_q_heads,
|
|
num_kv_heads=num_kv_heads,
|
|
head_dim=1,
|
|
)
|
|
return value[start:end].clone()
|
|
|
|
for suffix, kind in (
|
|
(".self_attn.q_proj.weight", "q"),
|
|
(".self_attn.q_proj.bias", "q"),
|
|
(".self_attn.k_proj.weight", "kv"),
|
|
(".self_attn.k_proj.bias", "kv"),
|
|
(".self_attn.v_proj.weight", "kv"),
|
|
(".self_attn.v_proj.bias", "kv"),
|
|
):
|
|
if name.endswith(suffix):
|
|
start, end = _attn_range(
|
|
kind=kind,
|
|
rank=rank,
|
|
world_size=world_size,
|
|
num_q_heads=num_q_heads,
|
|
num_kv_heads=num_kv_heads,
|
|
head_dim=head_dim,
|
|
)
|
|
return _shard_dim(value, 0, start, end)
|
|
|
|
if name.endswith(".self_attn.o_proj.weight"):
|
|
start, end = _attn_range(
|
|
kind="q",
|
|
rank=rank,
|
|
world_size=world_size,
|
|
num_q_heads=num_q_heads,
|
|
num_kv_heads=num_kv_heads,
|
|
head_dim=head_dim,
|
|
)
|
|
return _shard_dim(value, 1, start, end)
|
|
if name.endswith(".self_attn.o_proj.bias"):
|
|
return value.clone() if rank == 0 else torch.zeros_like(value)
|
|
|
|
if ".mlp.experts." in name:
|
|
tp_start, tp_end, local_intermediate = local_mxfp4_intermediate_range(
|
|
intermediate_size,
|
|
rank=rank,
|
|
world_size=world_size,
|
|
)
|
|
if name.endswith(".mlp.experts.gate_up_proj_blocks"):
|
|
return _shard_dim_pad(
|
|
value,
|
|
1,
|
|
2 * tp_start,
|
|
2 * tp_end,
|
|
2 * local_intermediate,
|
|
)
|
|
if name.endswith(".mlp.experts.gate_up_proj_scales"):
|
|
return _shard_dim_pad(
|
|
value,
|
|
1,
|
|
2 * tp_start,
|
|
2 * tp_end,
|
|
2 * local_intermediate,
|
|
)
|
|
if name.endswith(".mlp.experts.gate_up_proj_bias"):
|
|
return _shard_dim_pad(
|
|
value,
|
|
1,
|
|
2 * tp_start,
|
|
2 * tp_end,
|
|
2 * local_intermediate,
|
|
)
|
|
if name.endswith(".mlp.experts.down_proj_blocks"):
|
|
return _shard_dim_pad(
|
|
value,
|
|
2,
|
|
tp_start // 32,
|
|
tp_end // 32,
|
|
local_intermediate // 32,
|
|
)
|
|
if name.endswith(".mlp.experts.down_proj_scales"):
|
|
return _shard_dim_pad(
|
|
value,
|
|
value.ndim - 1,
|
|
tp_start // 32,
|
|
tp_end // 32,
|
|
local_intermediate // 32,
|
|
)
|
|
if name.endswith(".mlp.experts.down_proj_bias"):
|
|
return value.clone() if rank == 0 else torch.zeros_like(value)
|
|
|
|
if "embed_tokens" in name or "lm_head" in name:
|
|
return shard_tensor(
|
|
name,
|
|
value,
|
|
rank=rank,
|
|
world_size=world_size,
|
|
num_kv_heads=num_kv_heads,
|
|
)
|
|
return value.clone()
|
|
|
|
|
|
def iter_weights(
|
|
model_path: str,
|
|
device: torch.device,
|
|
*,
|
|
include_moe_experts: bool,
|
|
include_non_moe: bool,
|
|
) -> Iterator[tuple[str, torch.Tensor]]:
|
|
config = parse_config(cached_load_hf_config(model_path))
|
|
tp_info = get_tp_info()
|
|
|
|
def sharded_tensors() -> Iterator[tuple[str, torch.Tensor]]:
|
|
for file in iter_root_safetensor_files_from_index(model_path):
|
|
with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
|
|
for raw_name in f.keys():
|
|
is_expert = ".mlp.experts." in raw_name
|
|
if is_expert and not include_moe_experts:
|
|
continue
|
|
if not is_expert and not include_non_moe:
|
|
continue
|
|
raw = f.get_tensor(raw_name)
|
|
yield raw_name, shard_gpt_oss_tensor(
|
|
raw_name,
|
|
raw,
|
|
rank=tp_info.rank,
|
|
world_size=tp_info.size,
|
|
num_q_heads=config.num_qo_heads,
|
|
num_kv_heads=config.num_kv_heads,
|
|
head_dim=config.head_dim,
|
|
intermediate_size=config.moe_intermediate_size,
|
|
)
|
|
|
|
yield from iter_merged_tensors(
|
|
sharded_tensors(),
|
|
_MERGE_RULES,
|
|
model_name="gpt_oss",
|
|
)
|
|
|
|
|
|
def _expert_layer_and_name(key: str) -> tuple[int, str] | None:
|
|
parts = key.split(".")
|
|
if len(parts) < 6 or parts[0] != "model" or parts[1] != "layers":
|
|
return None
|
|
if parts[4] != "experts":
|
|
return None
|
|
try:
|
|
layer_id = int(parts[2])
|
|
except ValueError:
|
|
return None
|
|
return layer_id, parts[5]
|
|
|
|
|
|
def _read_safetensor_slice(
|
|
f,
|
|
name: str,
|
|
slices: tuple[slice, ...],
|
|
) -> torch.Tensor:
|
|
if all(s == slice(None) for s in slices):
|
|
return f.get_tensor(name)
|
|
return f.get_slice(name)[slices]
|
|
|
|
|
|
_EXPERT_SOURCES = (
|
|
"gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias",
|
|
"down_proj_blocks", "down_proj_scales", "down_proj_bias",
|
|
)
|
|
|
|
|
|
def _source_slices(config, tp_info) -> dict[str, tuple[str, tuple[slice, ...]]]:
|
|
"""HF expert tensor -> (piece role, this rank's slice of the intermediate axis)."""
|
|
tp_start, tp_end, _ = local_mxfp4_intermediate_range(
|
|
config.moe_intermediate_size, rank=tp_info.rank, world_size=tp_info.size
|
|
)
|
|
tp_end = max(tp_start, tp_end)
|
|
rows = slice(2 * tp_start, 2 * tp_end)
|
|
cols = slice(tp_start // 32, tp_end // 32)
|
|
return {
|
|
"gate_up_proj_blocks": ("gate_up", (slice(None), rows, slice(None), slice(None))),
|
|
"gate_up_proj_scales": ("gate_up_scale", (slice(None), rows, slice(None))),
|
|
"gate_up_proj_bias": ("gate_up_bias", (slice(None), rows)),
|
|
"down_proj_blocks": ("down", (slice(None), slice(None), cols, slice(None))),
|
|
"down_proj_scales": ("down_scale", (slice(None), slice(None), cols)),
|
|
"down_proj_bias": ("down_bias", (slice(None), slice(None))),
|
|
}
|
|
|
|
|
|
def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):
|
|
"""gpt-oss experts as HF ships them, one piece per layer: stacked ``[E, ...]`` mxfp4
|
|
``_blocks`` / ``_scales`` / ``_bias`` tensors sliced to this rank's intermediate range."""
|
|
if kind is not QuantKind.MXFP4:
|
|
return None
|
|
if config.moe_weight_format != "mxfp4":
|
|
raise ValueError("GPT-OSS offload requires MXFP4 expert weights")
|
|
tp_info = get_tp_info()
|
|
slices = _source_slices(config, tp_info)
|
|
num_layers, num_experts = config.num_layers, config.num_experts
|
|
keep = None if layers is None else set(layers)
|
|
|
|
def _is_expert(name: str) -> bool:
|
|
info = _expert_layer_and_name(name)
|
|
return info is not None and 0 <= info[0] < num_layers and (keep is None or info[0] in keep)
|
|
|
|
def _tensors():
|
|
if parallel:
|
|
from freetoken.models.weight import iter_expert_tensors_parallel
|
|
|
|
for name, whole in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):
|
|
_, source = _expert_layer_and_name(name)
|
|
yield name, whole[slices[source][1]]
|
|
return
|
|
for file in iter_root_safetensor_files_from_index(model_path):
|
|
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
|
|
for name in f.keys():
|
|
info = _expert_layer_and_name(name)
|
|
if info is None:
|
|
continue
|
|
layer_id, source = info
|
|
if not 0 <= layer_id < num_layers:
|
|
raise ValueError(f"Unexpected GPT-OSS expert layer in checkpoint: {name}")
|
|
if keep is not None and layer_id not in keep:
|
|
continue
|
|
if source not in slices:
|
|
raise ValueError(f"Unexpected GPT-OSS expert source: {name}")
|
|
yield name, _read_safetensor_slice(f, name, slices[source][1])
|
|
|
|
def _pieces():
|
|
pending: dict[int, dict[str, torch.Tensor]] = {}
|
|
for name, tensor in _tensors():
|
|
layer_id, source = _expert_layer_and_name(name)
|
|
role = slices[source][0]
|
|
if role == "down_bias" and tp_info.rank != 0:
|
|
tensor = torch.zeros_like(tensor) # the TP reduction adds the bias once, on rank 0
|
|
piece = pending.setdefault(layer_id, {})
|
|
piece[role] = tensor
|
|
if len(piece) == len(_EXPERT_SOURCES):
|
|
del pending[layer_id]
|
|
yield layer_id, 0, num_experts, piece
|
|
if pending:
|
|
raise ValueError(f"Missing GPT-OSS expert tensors for layers {sorted(pending)[:8]}")
|
|
|
|
return _pieces()
|
|
|
|
|
|
__all__ = [
|
|
"iter_weights",
|
|
"iter_expert_pieces",
|
|
"local_mxfp4_intermediate_range",
|
|
"shard_gpt_oss_tensor",
|
|
]
|