mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-23 02:00:02 +08:00
* feat(moe): serve experts of several formats through per-format slot pools * refactor(cpu-moe): pass each layer's cpu format with its options * fix(moe): read only each format's layers when loading experts of several formats --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
281 lines
11 KiB
Python
281 lines
11 KiB
Python
"""Weight loading for DeepSeek-V4-Flash (engine path).
|
|
|
|
- :func:`iter_weights` streams resident (non-expert) tensors keyed by the model's
|
|
attribute paths (``model.`` + the checkpoint name). ``wo_a`` dequantized to bf16 to match
|
|
the reference bf16 einsum.
|
|
- :func:`iter_expert_pieces` streams the routed MXFP4 experts (e2m1 pairs + e8m0 per-32
|
|
scales, no global) as per-expert pieces for the expert quant method's banks.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
|
|
import safetensors
|
|
import torch
|
|
from tqdm import tqdm
|
|
|
|
from freetoken.layers.quantization import QuantKind
|
|
from freetoken.distributed import get_tp_info
|
|
from freetoken.models.loader import drop_page_cache
|
|
|
|
from .args import DeepseekV4Args, load_args
|
|
|
|
|
|
class _ShardReader:
|
|
def __init__(self, folder: str, weight_map: dict, device):
|
|
self._folder = folder
|
|
self._weight_map = weight_map
|
|
self._device = str(device)
|
|
self._handles: dict[str, object] = {}
|
|
|
|
def has(self, name: str) -> bool:
|
|
return name in self._weight_map
|
|
|
|
def get(self, name: str) -> torch.Tensor:
|
|
shard = self._weight_map[name]
|
|
handle = self._handles.get(shard)
|
|
if handle is None:
|
|
handle = safetensors.safe_open(
|
|
os.path.join(self._folder, shard), framework="pt", device=self._device
|
|
).__enter__()
|
|
self._handles[shard] = handle
|
|
return handle.get_tensor(name)
|
|
|
|
def close(self) -> None:
|
|
for shard, handle in self._handles.items():
|
|
try:
|
|
handle.__exit__(None, None, None)
|
|
except Exception:
|
|
pass
|
|
drop_page_cache(os.path.join(self._folder, shard))
|
|
self._handles.clear()
|
|
|
|
|
|
def _weight_map(model_path: str) -> dict:
|
|
with open(os.path.join(model_path, "model.safetensors.index.json")) as f:
|
|
return json.load(f)["weight_map"]
|
|
|
|
|
|
def _dequant_fp8_block(weight: torch.Tensor, scale: torch.Tensor, block: int = 128) -> torch.Tensor:
|
|
"""Dequantize 128x128 block-scaled FP8 (e4m3) to bf16.
|
|
|
|
scale is e8m0 exponent codes, ``value = 2^(code-127)`` (Triton FP8 GEMM convention).
|
|
Used for ``wo_a`` to match the reference's bf16 einsum.
|
|
"""
|
|
n, k = weight.shape
|
|
codes = scale.view(torch.uint8).to(torch.float32)
|
|
s = torch.exp2(codes - 127.0)
|
|
s = s.repeat_interleave(block, dim=0).repeat_interleave(block, dim=1)[:n, :k]
|
|
return (weight.to(torch.float32) * s).to(torch.bfloat16)
|
|
|
|
|
|
def iter_weights(
|
|
model_path: str,
|
|
device,
|
|
*,
|
|
include_moe_experts: bool = True,
|
|
include_non_moe: bool = True,
|
|
include_vision: bool = True,
|
|
):
|
|
"""Stream resident (non-expert) weights as ``(name, tensor)`` keyed to engine params.
|
|
|
|
Routed MXFP4 experts come from the offload cache, so ``include_moe_experts`` must be
|
|
False (DeepSeek-V4 only runs ``--moe-strategy offload``). Tensors yielded in checkpoint
|
|
dtype (fp8 + e8m0 preserved); ``wo_a`` dequantized to bf16 to match the reference einsum.
|
|
``include_vision`` (the engine's active-encoder set) drops the ``visual.*`` tower
|
|
tensors; the router biases stay in either way, since the gate builds
|
|
them on any vision checkpoint.
|
|
"""
|
|
if include_moe_experts:
|
|
raise ValueError(
|
|
"DeepSeek-V4 routed experts are served from the offload cache; "
|
|
"run with --moe-strategy offload (include_moe_experts must be False)."
|
|
)
|
|
if not include_non_moe:
|
|
return
|
|
|
|
args = load_args(model_path, max_batch_size=1)
|
|
reader = _ShardReader(model_path, _weight_map(model_path), device)
|
|
|
|
def get(name: str) -> torch.Tensor:
|
|
return reader.get(name)
|
|
|
|
def linear(src: str, dst: str):
|
|
yield f"{dst}.weight", get(f"{src}.weight")
|
|
# fp8 linears declare the e8m0 block scale under the quant method's role name
|
|
if reader.has(f"{src}.scale"):
|
|
yield f"{dst}.weight_scale_inv", get(f"{src}.scale")
|
|
|
|
try:
|
|
yield "model.embed.weight", get("embed.weight")
|
|
yield "model.norm.weight", get("norm.weight")
|
|
yield "model.head.weight", get("head.weight")
|
|
for nm in ("hc_head_fn", "hc_head_base", "hc_head_scale"):
|
|
yield f"model.{nm}", get(nm)
|
|
|
|
if include_vision:
|
|
yield from _iter_vision(reader, args)
|
|
|
|
for L in range(args.n_layers):
|
|
a = f"layers.{L}.attn"
|
|
m = f"model.{a}"
|
|
yield from linear(f"{a}.wq_a", f"{m}.wq_a")
|
|
yield f"{m}.q_norm.weight", get(f"{a}.q_norm.weight")
|
|
yield from linear(f"{a}.wq_b", f"{m}.wq_b")
|
|
yield from linear(f"{a}.wkv", f"{m}.wkv")
|
|
yield f"{m}.kv_norm.weight", get(f"{a}.kv_norm.weight")
|
|
# wo_a: FP8 in the checkpoint, dequantized to bf16 (reference bf16 einsum).
|
|
yield f"{m}.wo_a", _dequant_fp8_block(
|
|
get(f"{a}.wo_a.weight"), get(f"{a}.wo_a.scale")
|
|
)
|
|
yield from linear(f"{a}.wo_b", f"{m}.wo_b")
|
|
yield f"{m}.attn_sink", get(f"{a}.attn_sink")
|
|
|
|
ratio = args.compress_ratios[L]
|
|
if ratio:
|
|
c = f"{a}.compressor"
|
|
for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):
|
|
yield f"model.{c}.{nm}", get(f"{c}.{nm}")
|
|
if ratio == 4:
|
|
idx = f"{a}.indexer"
|
|
yield from linear(f"{idx}.wq_b", f"model.{idx}.wq_b")
|
|
yield f"model.{idx}.weights_proj.weight", get(f"{idx}.weights_proj.weight")
|
|
ic = f"{idx}.compressor"
|
|
for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):
|
|
yield f"model.{ic}.{nm}", get(f"{ic}.{nm}")
|
|
|
|
yield f"model.layers.{L}.attn_norm.weight", get(f"layers.{L}.attn_norm.weight")
|
|
yield f"model.layers.{L}.ffn_norm.weight", get(f"layers.{L}.ffn_norm.weight")
|
|
|
|
g = f"layers.{L}.ffn.gate"
|
|
yield f"model.{g}.weight", get(f"{g}.weight")
|
|
if L < args.n_hash_layers:
|
|
yield f"model.{g}.tid2eid", get(f"{g}.tid2eid")
|
|
if args.vision_n_layers > 0:
|
|
# a vision checkpoint gives its hash layers a text bias too
|
|
yield f"model.{g}.bias", get(f"{g}.bias")
|
|
else:
|
|
yield f"model.{g}.bias", get(f"{g}.bias")
|
|
if args.vision_n_layers > 0:
|
|
yield f"model.{g}.bias_vl", get(f"{g}.bias_vl")
|
|
for proj in ("w1", "w2", "w3"):
|
|
src = f"layers.{L}.ffn.shared_experts.{proj}"
|
|
yield from linear(src, f"model.{src}")
|
|
|
|
for nm in (
|
|
"hc_attn_fn", "hc_ffn_fn", "hc_attn_base",
|
|
"hc_ffn_base", "hc_attn_scale", "hc_ffn_scale",
|
|
):
|
|
yield f"model.layers.{L}.{nm}", get(f"layers.{L}.{nm}")
|
|
finally:
|
|
reader.close()
|
|
|
|
|
|
# --------------------------------------------------------------------------------------
|
|
# Vision tower.
|
|
# --------------------------------------------------------------------------------------
|
|
# The wrapper mounts the tower, the aligner and the sentinels under one ``visual``
|
|
# container, so every engine-side name carries the ``visual.`` prefix the shared
|
|
# VISION_KEY_PREFIXES filter matches; the checkpoint's unprefixed names
|
|
# (``vision.*``, ``aligner.*``, ``image_*``) are renamed here on load.
|
|
_VISION_BLOCK_SUFFIXES = (
|
|
"norm1.weight",
|
|
"attn.wqkv.weight",
|
|
"attn.wqkv.bias",
|
|
"attn.wo.weight",
|
|
"attn.wo.bias",
|
|
"norm2.weight",
|
|
"mlp.w1.weight",
|
|
"mlp.w2.weight",
|
|
)
|
|
_VISION_TAIL = (
|
|
"vision.patch_embed.proj.weight",
|
|
"vision.patch_embed.proj.bias",
|
|
"vision.norm.weight",
|
|
"aligner.w1.weight",
|
|
"aligner.w1.bias",
|
|
"aligner.w2.weight",
|
|
"aligner.w2.bias",
|
|
"image_start",
|
|
"image_end",
|
|
"image_newline",
|
|
"image_pad",
|
|
)
|
|
|
|
|
|
def _iter_vision(reader, args: DeepseekV4Args):
|
|
if args.vision_n_layers <= 0:
|
|
return
|
|
for L in range(args.vision_n_layers):
|
|
for suffix in _VISION_BLOCK_SUFFIXES:
|
|
yield f"visual.vision.blocks.{L}.{suffix}", reader.get(f"vision.blocks.{L}.{suffix}")
|
|
for name in _VISION_TAIL:
|
|
yield f"visual.{name}", reader.get(name)
|
|
|
|
|
|
def iter_vision_weights(model_path: str, device):
|
|
"""The vision tower alone, for the engine's encoder-only load and the FTW converter."""
|
|
args = load_args(model_path, max_batch_size=1)
|
|
reader = _ShardReader(model_path, _weight_map(model_path), device)
|
|
try:
|
|
yield from _iter_vision(reader, args)
|
|
finally:
|
|
reader.close()
|
|
|
|
|
|
# --------------------------------------------------------------------------------------
|
|
# Routed MXFP4 expert pieces.
|
|
# --------------------------------------------------------------------------------------
|
|
_EXPERT_RE = re.compile(
|
|
r"^layers\.(?P<layer>\d+)\.ffn\.experts\.(?P<expert>\d+)\."
|
|
r"(?P<proj>w1|w2|w3)\.(?P<kind>weight|scale)$"
|
|
)
|
|
_PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
|
|
_KIND_SUFFIX = {"weight": "", "scale": "_scale"}
|
|
|
|
|
|
def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
|
|
"""Routed experts, one piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0
|
|
``_scale`` companions (``w1`` / ``w3`` / ``w2``). The MTP layer's experts are skipped."""
|
|
if kind is not QuantKind.MXFP4:
|
|
return None
|
|
if get_tp_info().size > 1:
|
|
raise NotImplementedError("DeepSeek-V4 expert banks support TP=1 only")
|
|
from freetoken.models.weight import iter_expert_tensors_parallel
|
|
from freetoken.moe.expert_pieces import per_expert_pieces
|
|
|
|
args = load_args(model_path, max_batch_size=1)
|
|
L, E = args.n_layers, args.n_routed_experts
|
|
keep = None if layers is None else set(layers)
|
|
|
|
def locate(raw_name: str):
|
|
m = _EXPERT_RE.match(raw_name)
|
|
if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):
|
|
return None
|
|
return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]
|
|
|
|
if parallel:
|
|
tensors = iter_expert_tensors_parallel(model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)
|
|
return per_expert_pieces(tensors, locate, tensors_per_expert=6)
|
|
|
|
def _serial():
|
|
reader = _ShardReader(model_path, _weight_map(model_path), torch.device("cpu"))
|
|
try:
|
|
for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):
|
|
for e in range(E):
|
|
base = f"layers.{li}.ffn.experts.{e}"
|
|
for proj in ("w1", "w3", "w2"):
|
|
for kind_ in ("weight", "scale"):
|
|
name = f"{base}.{proj}.{kind_}"
|
|
yield name, reader.get(name)
|
|
finally:
|
|
reader.close()
|
|
|
|
return per_expert_pieces(_serial(), locate, tensors_per_expert=6)
|
|
|
|
|
|
__all__ = ["iter_weights", "iter_expert_pieces", "iter_vision_weights"]
|