mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-23 02:00:02 +08:00
* feat(moe): serve experts of several formats through per-format slot pools * refactor(cpu-moe): pass each layer's cpu format with its options * fix(moe): read only each format's layers when loading experts of several formats --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
244 lines
12 KiB
Python
244 lines
12 KiB
Python
"""Weight loading for DeepSeek-V4.1-Flash.
|
|
|
|
* :func:`iter_weights` streams the resident (non-expert) tensors keyed by the model's attribute
|
|
paths: checkpoint ``layers.N.*`` -> ``model.layers.N.*``, ``embed`` / ``norm`` -> ``model.*``,
|
|
``head`` -> ``head``. fp8 linears keep their e4m3 payload and e8m0 ``scale`` (declared by the quant
|
|
method as ``weight_scale_inv``); ``wo_a`` keeps its e4m3 payload and ``wo_a_scale`` too (the layer
|
|
dequantizes in-kernel for the reference's bf16 grouped einsum). The engine casts everything else to
|
|
the declared parameter dtype (fp32 compressor weights, hc gates, engram gate weights).
|
|
* :func:`iter_expert_pieces` streams the routed MXFP4 experts (e2m1 pairs + e8m0 per-32 scales) per
|
|
expert for the offload banks.
|
|
|
|
Dropped: the DSpark ``mtp.*`` stage; vision weights and ``gate.bias_vl`` when serving text-only.
|
|
The Engram tables (``layers.{1,14}.engram.embed.*``) are
|
|
not state-dict tensors: the disk row store maps them in place (``engram_table.py``); an FTW conversion
|
|
carries them as side files (:func:`ftw_side_files`).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from contextlib import ExitStack
|
|
import json
|
|
import os
|
|
import re
|
|
|
|
import safetensors
|
|
import torch
|
|
from freetoken.distributed import get_tp_info
|
|
from freetoken.layers.quantization import QuantKind
|
|
from freetoken.models.loader import drop_page_cache
|
|
from tqdm import tqdm
|
|
|
|
from .args import DeepseekV41Args, Mode
|
|
|
|
|
|
class _ShardReader:
|
|
def __init__(self, folder: str, weight_map: dict, device):
|
|
self._folder = folder
|
|
self._weight_map = weight_map
|
|
self._device = str(device)
|
|
self._handles: dict[str, object] = {}
|
|
self._stack = ExitStack()
|
|
|
|
def has(self, name: str) -> bool:
|
|
return name in self._weight_map
|
|
|
|
def get(self, name: str) -> torch.Tensor:
|
|
shard = self._weight_map[name]
|
|
handle = self._handles.get(shard)
|
|
if handle is None:
|
|
handle = self._stack.enter_context(safetensors.safe_open(os.path.join(self._folder, shard), framework="pt", device=self._device))
|
|
self._handles[shard] = handle
|
|
return handle.get_tensor(name)
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
self._stack.close()
|
|
finally:
|
|
for shard in self._handles:
|
|
drop_page_cache(os.path.join(self._folder, shard))
|
|
self._handles.clear()
|
|
|
|
|
|
def weight_map(model_path: str) -> dict:
|
|
with open(os.path.join(model_path, "model.safetensors.index.json")) as f:
|
|
return json.load(f)["weight_map"]
|
|
|
|
|
|
def load_args(model_path: str) -> DeepseekV41Args:
|
|
from freetoken.utils.hf import cached_load_hf_config
|
|
|
|
return DeepseekV41Args.from_hf(cached_load_hf_config(model_path))
|
|
|
|
|
|
def dequant_fp8_block(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
|
"""Square block-scaled e4m3 -> bf16; ``scale`` holds e8m0 codes (``2^(code-127)``), the block
|
|
edge follows from the shapes."""
|
|
n, k = weight.shape
|
|
block = n // scale.shape[0]
|
|
assert (n // block, k // block) == tuple(scale.shape), (weight.shape, scale.shape)
|
|
s = torch.exp2(scale.view(torch.uint8).to(torch.float32) - 127.0)
|
|
s = s.repeat_interleave(block, dim=0).repeat_interleave(block, dim=1)
|
|
return (weight.to(torch.float32) * s).to(torch.bfloat16)
|
|
|
|
|
|
def iter_weights(model_path: str, device, *, include_moe_experts: bool = True, include_non_moe: bool = True, include_vision: bool = True):
|
|
args = load_args(model_path)
|
|
reader = _ShardReader(model_path, weight_map(model_path), device)
|
|
get = reader.get
|
|
def linear(src: str, dst: str):
|
|
yield f"{dst}.weight", get(f"{src}.weight")
|
|
if reader.has(f"{src}.scale"):
|
|
yield f"{dst}.weight_scale_inv", get(f"{src}.scale")
|
|
|
|
try:
|
|
if include_moe_experts:
|
|
# Resident experts support unquantized checkpoints; FP4 uses the offload cache.
|
|
if reader.has("layers.0.ffn.experts.0.w1.scale"):
|
|
raise ValueError(
|
|
"DeepSeek-V4.1's fp4 routed experts are served from the offload cache; run with "
|
|
"--moe-strategy offload (include_moe_experts must be False)."
|
|
)
|
|
for L in range(args.n_layers):
|
|
p = f"layers.{L}.ffn.experts"
|
|
gate_up = torch.stack([torch.cat([get(f"{p}.{e}.w1.weight"), get(f"{p}.{e}.w3.weight")]) for e in range(args.n_routed_experts)])
|
|
down = torch.stack([get(f"{p}.{e}.w2.weight") for e in range(args.n_routed_experts)])
|
|
yield f"model.layers.{L}.ffn.experts.gate_up_proj", gate_up
|
|
yield f"model.layers.{L}.ffn.experts.down_proj", down
|
|
if not include_non_moe:
|
|
return
|
|
if include_vision and reader.has("vision.patch_embed.proj.weight"):
|
|
yield from iter_vision_weights(model_path, device)
|
|
yield "model.embed.weight", get("embed.weight")
|
|
yield "model.norm.weight", get("norm.weight")
|
|
yield "head.weight", get("head.weight")
|
|
for L in range(args.n_layers):
|
|
role = args.roles[L]
|
|
a, m = f"layers.{L}.attn", f"model.layers.{L}.attn"
|
|
yield from linear(f"{a}.wq_a", f"{m}.wq_a")
|
|
yield f"{m}.q_norm.weight", get(f"{a}.q_norm.weight")
|
|
yield from linear(f"{a}.wq_b", f"{m}.wq_b")
|
|
yield from linear(f"{a}.wkv", f"{m}.wkv")
|
|
yield f"{m}.kv_norm.weight", get(f"{a}.kv_norm.weight")
|
|
yield f"{m}.wo_a", get(f"{a}.wo_a.weight")
|
|
if reader.has(f"{a}.wo_a.scale"): # the fp8 payload + scale; the layer dequantizes in-kernel
|
|
yield f"{m}.wo_a_scale", get(f"{a}.wo_a.scale")
|
|
yield from linear(f"{a}.wo_b", f"{m}.wo_b")
|
|
yield f"{m}.attn_sink", get(f"{a}.attn_sink")
|
|
if role.mode is Mode.FULL:
|
|
c = f"{a}.compressor"
|
|
yield f"{m}.compressor.norm.weight", get(f"{c}.norm.weight")
|
|
if role.ratio > 1:
|
|
yield f"{m}.compressor.wkv", get(f"{c}.wkv.weight")
|
|
yield f"{m}.compressor.wgate", get(f"{c}.wgate.weight")
|
|
else:
|
|
yield from linear(f"{c}.wkv", f"{m}.compressor.wkv")
|
|
yield from linear(f"{a}.indexer.wk", f"{m}.indexer.wk")
|
|
yield f"{m}.indexer.k_norm.weight", get(f"{a}.indexer.k_norm.weight")
|
|
if role.has_indexer:
|
|
yield from linear(f"{a}.indexer.wq_b", f"{m}.indexer.wq_b")
|
|
yield from linear(f"{a}.indexer.weights_proj", f"{m}.indexer.weights_proj")
|
|
|
|
p = f"layers.{L}"
|
|
yield f"model.{p}.attn_norm.weight", get(f"{p}.attn_norm.weight")
|
|
yield f"model.{p}.ffn_norm.weight", get(f"{p}.ffn_norm.weight")
|
|
yield f"model.{p}.ffn.gate.weight", get(f"{p}.ffn.gate.weight")
|
|
yield f"model.{p}.ffn.gate.bias", get(f"{p}.ffn.gate.bias")
|
|
for proj in ("w1", "w2", "w3"):
|
|
yield from linear(f"{p}.ffn.shared_experts.{proj}", f"model.{p}.ffn.shared_experts.{proj}")
|
|
for nm in ("hc_attn_fn", "hc_ffn_fn", "hc_attn_base", "hc_ffn_base", "hc_attn_scale", "hc_ffn_scale"):
|
|
yield f"model.{p}.{nm}", get(f"{p}.{nm}")
|
|
if L in args.engram_layer_ids:
|
|
e = f"{p}.engram"
|
|
yield from linear(f"{e}.wkv", f"model.{e}.wkv")
|
|
yield f"model.{e}.q_weight", get(f"{e}.q_weight")
|
|
yield f"model.{e}.k_weight", get(f"{e}.k_weight")
|
|
finally:
|
|
reader.close()
|
|
|
|
|
|
def iter_vision_weights(model_path: str, device):
|
|
names = weight_map(model_path)
|
|
reader = _ShardReader(model_path, names, device)
|
|
try:
|
|
for name in names:
|
|
if name.startswith(("vision.", "aligner.")) or name in ("image_start", "image_end", "image_newline"):
|
|
yield f"visual.{name}", reader.get(name)
|
|
args = load_args(model_path)
|
|
yield "visual.routing_bias", torch.stack([reader.get(f"layers.{i}.ffn.gate.bias_vl") for i in range(args.n_layers)])
|
|
finally:
|
|
reader.close()
|
|
|
|
|
|
# ----- FTW side files: the Engram tables ----------------------------------------------------------
|
|
def engram_table_names(layer_id: int) -> tuple[str, str]:
|
|
return f"layers.{layer_id}.engram.embed.weight", f"layers.{layer_id}.engram.embed.scale"
|
|
|
|
|
|
def ftw_side_files(model_path: str, out_dir: str) -> list[str]:
|
|
"""Copy the checkpoint shards holding the Engram tables (fp8 rows + ue8m0 scales) next to an FTW
|
|
checkpoint. The tables are never FTW entries: ``load_host_tables`` maps them in place from whatever
|
|
safetensors file holds them (``engram_table.engram_row_source`` reads the headers), and the FTW
|
|
loader never reads ``.safetensors`` as weights, so the shard's other tensors ride along unused."""
|
|
import shutil
|
|
|
|
from freetoken.models.loader import safetensors_weight_map
|
|
from freetoken.utils import download_hf_weight
|
|
|
|
folder = download_hf_weight(model_path)
|
|
args = load_args(folder)
|
|
shard_of = safetensors_weight_map(folder)
|
|
shards = sorted({shard_of[n] for layer_id in args.engram_layer_ids if layer_id < args.n_layers for n in engram_table_names(layer_id)})
|
|
for shard in shards:
|
|
shutil.copyfile(os.path.join(folder, shard), os.path.join(out_dir, shard))
|
|
return shards
|
|
|
|
|
|
# ----- routed MXFP4 expert pieces ---------------------------------------------------------------
|
|
_EXPERT_RE = re.compile(r"^layers\.(?P<layer>\d+)\.ffn\.experts\.(?P<expert>\d+)\.(?P<proj>w1|w2|w3)\.(?P<kind>weight|scale)$")
|
|
_PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
|
|
_KIND_SUFFIX = {"weight": "", "scale": "_scale"}
|
|
|
|
|
|
def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
|
|
"""One piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0 ``_scale`` companions.
|
|
The DSpark stage's experts (``mtp.*``) never match."""
|
|
if kind is not QuantKind.MXFP4:
|
|
return None
|
|
if get_tp_info().size > 1:
|
|
raise NotImplementedError("DeepSeek-V4.1 expert banks support TP=1 only")
|
|
from freetoken.models.weight import iter_expert_tensors_parallel
|
|
from freetoken.moe.expert_pieces import per_expert_pieces
|
|
|
|
args = load_args(model_path)
|
|
L, E = args.n_layers, args.n_routed_experts
|
|
keep = None if layers is None else set(layers)
|
|
|
|
def locate(raw_name: str):
|
|
m = _EXPERT_RE.match(raw_name)
|
|
if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):
|
|
return None
|
|
return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]
|
|
|
|
if parallel:
|
|
tensors = iter_expert_tensors_parallel(model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)
|
|
return per_expert_pieces(tensors, locate, tensors_per_expert=6)
|
|
|
|
def _serial():
|
|
reader = _ShardReader(model_path, weight_map(model_path), torch.device("cpu"))
|
|
try:
|
|
for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DeepSeek-V4.1 experts (serial)", disable=not get_tp_info().is_primary()):
|
|
for e in range(E):
|
|
base = f"layers.{li}.ffn.experts.{e}"
|
|
for proj in ("w1", "w3", "w2"):
|
|
for kind_ in ("weight", "scale"):
|
|
name = f"{base}.{proj}.{kind_}"
|
|
yield name, reader.get(name)
|
|
finally:
|
|
reader.close()
|
|
|
|
return per_expert_pieces(_serial(), locate, tensors_per_expert=6)
|
|
|
|
|
|
__all__ = ["iter_weights", "iter_vision_weights", "iter_expert_pieces", "ftw_side_files", "load_args", "weight_map", "dequant_fp8_block"]
|