Files
Jesús Jódar Piernas 2d93085545 feat(flint): add nvfp4 modelopt and gguf support (#797)
* feat(flint): add nvfp4 modelopt and gguf support

* refactor(flint): address final review scope requests
2026-10-24 12:37:57 -07:00

78 lines
3.3 KiB
Python

from __future__ import annotations
import re
from typing import Iterator
import safetensors
import torch
from freetoken.distributed import get_tp_info
from freetoken.models.loader import (
MergeRule, drop_page_cache, iter_merged_tensors, iter_stacked_experts,
iter_weight_files, safetensors_weight_map, shard_tensor,
)
from freetoken.models.nvfp4_banks import Nvfp4ExpertSourceSpec
from freetoken.utils import cached_load_hf_config, download_hf_weight
from .config import parse_config
_EXPERT = re.compile(r"^(?P<prefix>model\.layers\.\d+\.mlp\.experts)\.(?P<idx>\d+)\.(?P<name>.+)$")
_MERGE = {
".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
".gate_proj": MergeRule(".gate_up_proj", "gate", ("gate", "up")),
".up_proj": MergeRule(".gate_up_proj", "up", ("gate", "up")),
}
_NVFP4 = Nvfp4ExpertSourceSpec(
key_pattern=re.compile(
r"^model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."
r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"
),
proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},
layer_to_bank=lambda layer, config: (
layer - config.first_k_dense_replace if config.first_k_dense_replace <= layer < config.num_layers else None
),
desc="Flint NVFP4 experts",
)
def nvfp4_expert_spec(model_path: str, config) -> Nvfp4ExpertSourceSpec:
return _NVFP4
def iter_weights(
model_path: str, device: torch.device, *, include_moe_experts: bool, include_non_moe: bool,
) -> Iterator[tuple[str, torch.Tensor]]:
model_path = download_hf_weight(model_path)
config = parse_config(cached_load_hf_config(model_path))
if include_moe_experts and config.expert_quant != "none":
raise ValueError("Flint NVFP4 uses the offload, cpu or hybrid MoE strategy")
tp = get_tp_info()
names = safetensors_weight_map(model_path)
def tensors():
for file in iter_weight_files(model_path):
with safetensors.safe_open(file, framework="pt", device=str(device)) as reader:
for name in reader.keys():
expert = _EXPERT.match(name) is not None
if (expert and not include_moe_experts) or (not expert and not include_non_moe):
continue
raw = reader.get_tensor(name)
if name.endswith(".softmax_offset"):
raw = raw.chunk(tp.size)[tp.rank].float()
elif name.endswith(".expert_bias"):
raw = raw.float()
else:
raw = shard_tensor(name, raw, rank=tp.rank, world_size=tp.size, num_kv_heads=config.num_kv_heads)
yield name, raw
drop_page_cache(file)
if include_non_moe:
for layer in range(config.first_k_dense_replace, config.num_layers):
name = f"model.layers.{layer}.mlp.gate.expert_bias"
if name not in names:
yield name, torch.zeros(config.num_experts, dtype=torch.float32, device=device)
merged = iter_merged_tensors(tensors(), _MERGE, model_name="flint")
yield from iter_stacked_experts(merged, num_experts=config.num_experts, model_name="flint", expert_pattern=_EXPERT)