mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
NVFP4 weight global scales (amax / 2688, 2e-5 to 4e-4 in the served checkpoints) sit at fp16's normal minimum; stored as fp16, up to 97% of a checkpoint's expert globals were subnormal, with up to 1.7e-3 relative error per row. BREAKING CHANGE: an FTW checkpoint that stores NVFP4 global scales as fp16 is refused at load; reconvert it with `ft checkpoint`.
307 lines
13 KiB
Python
307 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Iterator
|
|
|
|
import safetensors
|
|
import torch
|
|
from freetoken.distributed import get_tp_info
|
|
from freetoken.models.config import FullAttentionGroupConfig
|
|
from freetoken.models.loader import (
|
|
MergeRule,
|
|
ShardReader,
|
|
drop_page_cache,
|
|
iter_weight_files,
|
|
)
|
|
from freetoken.models.nvfp4_banks import (
|
|
Nvfp4ExpertSourceSpec,
|
|
)
|
|
from freetoken.utils import cached_load_hf_config
|
|
from tqdm import tqdm
|
|
|
|
from .config import parse_config
|
|
|
|
_PACKED_EXPERT_PATTERN = re.compile(
|
|
r"^model\.layers\.(?P<layer>\d+)\.feed_forward\.experts\.(?P<name>gate_up_proj|down_proj)$"
|
|
)
|
|
|
|
# NVFP4 routed experts (nvidia modelopt checkpoint): per-expert, un-fused, under the raw
|
|
# ``model.language_model.layers.N.experts.E.{proj}`` key (no .mlp./.feed_forward. infix).
|
|
# Matched against the RAW weight_map key in nvfp4_banks (it never sees the renamed key).
|
|
_NVFP4_EXPERT_RE = re.compile(r"\.experts\.\d+\.")
|
|
_NVFP4_EXPERT_KEY_RE = re.compile(
|
|
r"^model\.language_model\.layers\.(?P<layer>\d+)\.experts\.(?P<expert>\d+)\."
|
|
r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"
|
|
)
|
|
_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(
|
|
key_pattern=_NVFP4_EXPERT_KEY_RE,
|
|
proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},
|
|
layer_to_bank=lambda layer, config: layer, # every layer is MoE (no dense prefix)
|
|
desc="Gemma4 NVFP4 experts",
|
|
)
|
|
_LAYER_INDEX_PATTERN = re.compile(r"layers\.(\d+)\.")
|
|
_LAYER_FF_PREFIX_PATTERN = re.compile(r"^(model\.layers\.\d+)\.")
|
|
_MERGE_RULES = {
|
|
".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
|
|
".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
|
|
".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
|
|
".gate_proj": MergeRule(".gate_up_proj", "gate", ("gate", "up")),
|
|
".up_proj": MergeRule(".gate_up_proj", "up", ("gate", "up")),
|
|
}
|
|
_FEED_FORWARD_PREFIXES = (
|
|
"experts.",
|
|
"router.",
|
|
"layer_scalar",
|
|
"post_feedforward_layernorm.",
|
|
"post_feedforward_layernorm_1.",
|
|
"post_feedforward_layernorm_2.",
|
|
"pre_feedforward_layernorm_2.",
|
|
)
|
|
|
|
# modelopt-NVFP4 dense MLP (nvidia/Gemma-4-31B-IT-NVFP4): mlp.{gate,up,down}_proj are W4A16
|
|
# FP4 -- uint8 weight + fp8-e4m3 block weight_scale + per-tensor weight_scale_2 + input_scale.
|
|
# The scales are consumed with their .weight.
|
|
_NVFP4_DENSE_SCALE_SUFFIXES = (".weight_scale", ".weight_scale_2", ".input_scale")
|
|
_NVFP4_DENSE_MLP_RE = re.compile(r"\.mlp\.(gate_proj|up_proj|down_proj)\.weight$")
|
|
|
|
|
|
def _nvfp4_dense_parts(reader: ShardReader, raw_base: str):
|
|
"""Load an NVFP4 dense weight as the NVFP4 linear method's buffers: weight uint8 [O, IN//2], weight_scale fp8-e4m3 block [O, IN//16], weight_global fp32 [O] (the per-tensor weight_scale_2 per output row), input_scale fp32 scalar or None when the export has none."""
|
|
w = reader.get_tensor(raw_base + ".weight")
|
|
s = reader.get_tensor(raw_base + ".weight_scale")
|
|
g = reader.get_tensor(raw_base + ".weight_scale_2").reshape(1).to(torch.float32)
|
|
g = g.expand(w.shape[0]).contiguous()
|
|
assert (
|
|
w.dtype is torch.uint8
|
|
and s.dtype is torch.float8_e4m3fn
|
|
and g.dtype is torch.float32
|
|
), f"unexpected NVFP4 dense dtypes at {raw_base}: {w.dtype}/{s.dtype}/{g.dtype}"
|
|
a = reader.get_tensor(raw_base + ".input_scale").reshape(()).to(torch.float32) if reader.has(raw_base + ".input_scale") else None
|
|
return w, s, g, a
|
|
|
|
|
|
def _emit_nvfp4_dense_mlp(reader: ShardReader, base: str, raw_base: str, buf: dict):
|
|
"""(key, tensor) pairs for an NVFP4 dense MLP projection: down_proj standalone;
|
|
gate_proj/up_proj merged output-wise into gate_up_proj (each keeps its own scales, so the
|
|
fused weight is exact). Returns [] while a gate/up merge is still buffered."""
|
|
w, s, g, a = _nvfp4_dense_parts(reader, raw_base)
|
|
if base.endswith(".down_proj"):
|
|
out = [(base + ".weight", w), (base + ".weight_scale", s), (base + ".weight_global", g)]
|
|
return out + ([(base + ".input_scale", a)] if a is not None else [])
|
|
is_gate = base.endswith(".gate_proj")
|
|
prefix = base[: -len(".gate_proj")] if is_gate else base[: -len(".up_proj")]
|
|
slots = buf.setdefault(prefix, {})
|
|
slots["gate" if is_gate else "up"] = (w, s, g, a)
|
|
if "gate" not in slots or "up" not in slots:
|
|
return []
|
|
gw, gs, gg, ga = slots["gate"]
|
|
uw, us, ug, ua = slots["up"]
|
|
del buf[prefix]
|
|
pre = prefix + ".gate_up_proj"
|
|
out = [
|
|
(pre + ".weight", torch.cat([gw, uw], dim=0)),
|
|
(pre + ".weight_scale", torch.cat([gs, us], dim=0)),
|
|
(pre + ".weight_global", torch.cat([gg, ug], dim=0)),
|
|
]
|
|
if ga is not None and ua is not None:
|
|
# both parts read the same activation; the larger range covers both
|
|
out.append((pre + ".input_scale", torch.maximum(ga, ua)))
|
|
return out
|
|
|
|
|
|
def _rename_vision_key(raw_name: str) -> str | None:
|
|
"""Encoder checkpoint key -> state-dict key, or None for anything else; the tower's clipped-linear wrapper is dropped."""
|
|
if raw_name.startswith("model.vision_tower."):
|
|
return ("vision_tower." + raw_name[len("model.vision_tower.") :]).replace(".linear.", ".")
|
|
if raw_name.startswith("model.embed_vision."):
|
|
return "embed_vision." + raw_name[len("model.embed_vision.") :]
|
|
if raw_name.startswith("model.vision_embedder."):
|
|
return "vision_embedder." + raw_name[len("model.vision_embedder.") :]
|
|
return None
|
|
|
|
|
|
def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
|
|
"""The vision encoder alone, named as iter_weights names it."""
|
|
for file in iter_weight_files(model_path):
|
|
with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
|
|
for raw_name in f.keys():
|
|
name = _rename_vision_key(raw_name)
|
|
if name is not None:
|
|
yield name, f.get_tensor(raw_name)
|
|
|
|
|
|
def _rename_language_key(raw_name: str) -> str:
|
|
name = raw_name.removeprefix("model.language_model.")
|
|
name = "model." + name.removeprefix("language_model.")
|
|
|
|
match = _LAYER_FF_PREFIX_PATTERN.match(name)
|
|
if match is None:
|
|
return name
|
|
|
|
layer_prefix = match.group(1)
|
|
layer_key = name[match.end() :]
|
|
if layer_key.startswith("mlp."):
|
|
return f"{layer_prefix}.feed_forward.shared_mlp.{layer_key.removeprefix('mlp.')}"
|
|
if layer_key.startswith(_FEED_FORWARD_PREFIXES):
|
|
return f"{layer_prefix}.feed_forward.{layer_key}"
|
|
return name
|
|
|
|
|
|
def iter_weights(
|
|
model_path: str,
|
|
device: torch.device,
|
|
*,
|
|
include_moe_experts: bool,
|
|
include_non_moe: bool,
|
|
include_vision: bool = True,
|
|
) -> Iterator[tuple[str, torch.Tensor]]:
|
|
def rename_key(raw_name: str) -> str | None:
|
|
prefix = "model.language_model."
|
|
if raw_name.startswith(prefix):
|
|
return _rename_language_key(raw_name)
|
|
if raw_name.startswith("language_model."):
|
|
return _rename_language_key(raw_name)
|
|
return _rename_vision_key(raw_name)
|
|
|
|
def merge_info(key: str) -> tuple[str, MergeRule] | None:
|
|
for suffix, rule in _MERGE_RULES.items():
|
|
if key.endswith(suffix + ".weight") or key.endswith(suffix):
|
|
return key.replace(suffix, rule.fused_suffix), rule
|
|
return None
|
|
|
|
config = parse_config(cached_load_hf_config(model_path))
|
|
tp_info = get_tp_info()
|
|
if tp_info.size > 1:
|
|
raise NotImplementedError("Gemma 4 weight loading currently supports TP=1 only")
|
|
|
|
k_eq_v_layers = {
|
|
layer_id
|
|
for layer_id in range(config.num_layers)
|
|
if isinstance(config.attention_group_for_layer(layer_id), FullAttentionGroupConfig)
|
|
and config.attention_group_for_layer(layer_id).k_eq_v
|
|
}
|
|
merge_buf: dict[str, dict[str, torch.Tensor]] = {}
|
|
gateup_buf: dict[str, dict[str, tuple]] = {}
|
|
reader = ShardReader(model_path, device)
|
|
try:
|
|
for file in tqdm(
|
|
iter_weight_files(model_path),
|
|
desc="Loading weights",
|
|
disable=not tp_info.is_primary(),
|
|
):
|
|
with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
|
|
for raw_name in f.keys():
|
|
name = rename_key(raw_name)
|
|
if name is None:
|
|
continue
|
|
|
|
# Per-expert NVFP4 tensors go to the offload cache (expert pieces),
|
|
# not this dense pass; fused bf16 experts lack ".experts.<int>." so are unaffected.
|
|
if _NVFP4_EXPERT_RE.search(raw_name):
|
|
continue
|
|
|
|
# NVFP4 dense-MLP scales are consumed with their .weight (below), never yielded.
|
|
if raw_name.endswith(_NVFP4_DENSE_SCALE_SUFFIXES):
|
|
continue
|
|
|
|
is_vision = name.startswith(("vision_tower.", "embed_vision.", "vision_embedder."))
|
|
if is_vision and not include_vision:
|
|
continue
|
|
is_expert = (
|
|
not is_vision and _PACKED_EXPERT_PATTERN.match(name) is not None
|
|
)
|
|
if is_expert and not include_moe_experts:
|
|
continue
|
|
if not is_expert and not include_non_moe:
|
|
continue
|
|
|
|
# Native W4A16 NVFP4 dense MLP: the .weight is FP4-packed and carries block + per-tensor scales.
|
|
# The weight_scale_2 sibling guard is defense-in-depth beyond config.dense_quant -- the sibling MoE checkpoint's bf16 shared_mlp has no such sibling, so it falls through to the bf16 path.
|
|
if (
|
|
config.dense_quant == "nvfp4"
|
|
and not is_vision
|
|
and not is_expert
|
|
and _NVFP4_DENSE_MLP_RE.search(raw_name)
|
|
and reader.has(raw_name[: -len(".weight")] + ".weight_scale_2")
|
|
):
|
|
yield from _emit_nvfp4_dense_mlp(
|
|
reader, name[: -len(".weight")], raw_name[: -len(".weight")], gateup_buf
|
|
)
|
|
continue
|
|
|
|
tensor = f.get_tensor(raw_name)
|
|
if is_vision or is_expert:
|
|
yield name, tensor
|
|
continue
|
|
|
|
info = merge_info(name)
|
|
if info is None:
|
|
yield name, tensor
|
|
continue
|
|
|
|
merged_key, rule = info
|
|
slots = merge_buf.setdefault(merged_key, {})
|
|
slots[rule.slot] = tensor
|
|
if rule.slot == "k" and k_eq_v_layers:
|
|
layer_match = _LAYER_INDEX_PATTERN.search(name)
|
|
if (
|
|
layer_match is not None
|
|
and int(layer_match.group(1)) in k_eq_v_layers
|
|
):
|
|
slots["v"] = tensor
|
|
if not all(slot in slots for slot in rule.slots):
|
|
continue
|
|
parts = [slots[slot] for slot in rule.slots]
|
|
del merge_buf[merged_key]
|
|
yield merged_key, torch.cat(parts, dim=0)
|
|
|
|
finally:
|
|
reader.close()
|
|
|
|
assert not merge_buf, f"Incomplete merge groups in checkpoint: {list(merge_buf.keys())}"
|
|
assert not gateup_buf, f"Incomplete NVFP4 gate/up merges: {list(gateup_buf.keys())}"
|
|
|
|
|
|
def iter_weights_parallel(
|
|
model_path: str,
|
|
device: torch.device,
|
|
*,
|
|
include_moe_experts: bool,
|
|
include_non_moe: bool,
|
|
workers: int = 8,
|
|
chunk: int = 8 << 20,
|
|
) -> Iterator[tuple[str, torch.Tensor]]:
|
|
"""experts-only parallel reader: gemma-4 packs experts per-layer
|
|
(``feed_forward.experts.{gate_up_proj,down_proj}``), so no merge needed; same key
|
|
rename as iter_weights, read via the common chunked O_DIRECT reader."""
|
|
assert include_moe_experts and not include_non_moe, (
|
|
"gemma4 parallel reader is experts-only (used by the expert piece reader)"
|
|
)
|
|
from freetoken.models.weight import iter_expert_tensors_parallel
|
|
|
|
if get_tp_info().size > 1:
|
|
raise NotImplementedError("Gemma 4 weight loading currently supports TP=1 only")
|
|
|
|
def _expert_name(raw_name: str) -> str | None:
|
|
if raw_name.startswith("model.language_model.") or raw_name.startswith("language_model."):
|
|
name = _rename_language_key(raw_name)
|
|
if _PACKED_EXPERT_PATTERN.match(name) is not None:
|
|
return name
|
|
return None
|
|
|
|
for raw_name, tensor in iter_expert_tensors_parallel(
|
|
model_path, lambda rn: _expert_name(rn) is not None, workers=workers, chunk=chunk
|
|
):
|
|
yield _expert_name(raw_name), tensor
|
|
|
|
|
|
def nvfp4_expert_spec(model_path: str, config):
|
|
return _NVFP4_SOURCE_SPEC
|
|
|
|
|
|
__all__ = [
|
|
"nvfp4_expert_spec",
|
|
"iter_weights",
|
|
"iter_weights_parallel",
|
|
]
|