mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-23 02:00:02 +08:00
255 lines
9.8 KiB
Python
255 lines
9.8 KiB
Python
"""Engine-facing config for GLM-5.3-Flash (``glm5_next``).
|
|
|
|
Two attention groups, ordered by first layer id:
|
|
|
|
* ``linear`` -- 34 KDA layers (``LinearGatedDeltaGroupConfig`` with
|
|
``variant="kda"``). KDA's state geometry coincides with GDN's (conv width
|
|
``2*K*dk + V*dv`` == q|k|v at ``H*d`` each; recurrent ``[H, d, d]``), so the
|
|
existing ``LinearStatePool`` shapes serve it unchanged.
|
|
* ``full`` -- 11 DSA layers (``FullAttentionGroupConfig`` with ``mla=True`` and
|
|
the indexer dims). The MLA is NoPE (``qk_rope_head_dim == 0``): the latent row
|
|
is bare ckv (512), and the indexer K slab is kpool-compressed
|
|
(``index_kpool=4`` -> tokens/4 stored entries; the spec's ``index_ratio``
|
|
drives both the pool factory and the KV cost model).
|
|
|
|
MoE routing mirrors glm_moe_dsa (sigmoid ``noaux_tc``, shared expert, routed
|
|
scaling 2.5) with 288 routed experts and a clamped SwiGLU
|
|
(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in
|
|
``glm5_args`` (``Glm5NextArgs``).
|
|
|
|
The checkpoint is served in the precision its quantization_config declares.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
from freetoken.models.config import (
|
|
FullAttentionGroupConfig,
|
|
LinearGatedDeltaGroupConfig,
|
|
ModelConfig,
|
|
RotaryConfig,
|
|
detect_expert_quant,
|
|
)
|
|
|
|
from .args import load_args
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VisionConfig:
|
|
hidden_size: int
|
|
depth: int
|
|
num_heads: int
|
|
intermediate_size: int
|
|
projection_intermediate_size: int
|
|
out_hidden_size: int
|
|
in_channels: int
|
|
patch_size: int
|
|
temporal_patch_size: int
|
|
spatial_merge_size: int
|
|
rms_norm_eps: float
|
|
swiglu_limit: float
|
|
attention_bias: bool
|
|
|
|
|
|
def parse_vision_config(hf_config: Any) -> VisionConfig | None:
|
|
"""None when the config carries no vision section, which is how a text-only engine asks for no tower."""
|
|
vc = getattr(hf_config, "vision_config", None)
|
|
if vc is None:
|
|
return None
|
|
if vc.hidden_act != "silu":
|
|
raise NotImplementedError(f"glm5_next vision tower activation {vc.hidden_act!r}; only silu is implemented")
|
|
return VisionConfig(
|
|
hidden_size=vc.hidden_size,
|
|
depth=vc.depth,
|
|
num_heads=vc.num_heads,
|
|
intermediate_size=vc.intermediate_size,
|
|
projection_intermediate_size=vc.projection_intermediate_size,
|
|
out_hidden_size=vc.out_hidden_size,
|
|
in_channels=vc.in_channels,
|
|
patch_size=vc.patch_size,
|
|
temporal_patch_size=vc.temporal_patch_size,
|
|
spatial_merge_size=vc.spatial_merge_size,
|
|
rms_norm_eps=vc.rms_norm_eps,
|
|
swiglu_limit=vc.swiglu_limit,
|
|
attention_bias=bool(vc.attention_bias),
|
|
)
|
|
|
|
|
|
def _dsa_on(args, dsa_layer_ids) -> bool:
|
|
"""DSA serving switch, resolved ONCE into the attention-group spec (the pool
|
|
factory, KV cost model, and backend read the spec, never the env)."""
|
|
return (
|
|
len(dsa_layer_ids) > 0
|
|
and args.index_topk > 0
|
|
and args.index_head_dim > 0
|
|
and os.getenv("FREETOKEN_GLM5_DSA", "1") != "0"
|
|
)
|
|
|
|
|
|
def _quant_accessor(hf_config: Any):
|
|
"""A ``get(key, default=None)`` accessor over the HF ``quantization_config`` (dict or
|
|
object), or ``None`` when the model has no quant config."""
|
|
quant = getattr(hf_config, "quantization_config", None)
|
|
if quant is None:
|
|
return None
|
|
return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
|
|
|
|
|
|
def _fp8_block_quant(hf_config: Any) -> tuple[str, tuple[int, int] | None]:
|
|
"""Detect DeepSeek-V3-style 128x128 block-fp8 from HF ``quantization_config``.
|
|
|
|
Returns ``("fp8_block", (block_n, block_k))`` for a block-fp8 checkpoint (weights
|
|
fp8-e4m3 + per-block ``weight_scale_inv``, dynamic activation), else ``("none", None)``.
|
|
The quantization_config sits on the top-level hf_config (not ``text_config``).
|
|
"""
|
|
get = _quant_accessor(hf_config)
|
|
if get is None:
|
|
return "none", None
|
|
method = str(get("quant_method") or get("quant_algo") or "").lower()
|
|
block = get("weight_block_size")
|
|
if method == "fp8" and block:
|
|
bs = tuple(int(x) for x in block)
|
|
assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"
|
|
return "fp8_block", bs
|
|
return "none", None
|
|
|
|
|
|
def parse_config(hf_config: Any) -> ModelConfig:
|
|
args = load_args(hf_config)
|
|
text = getattr(hf_config, "text_config", hf_config)
|
|
|
|
num_layers = len(args.layer_types)
|
|
# Dev/testing only: cap the layer count so the forward path / KV / offload cache
|
|
# can be exercised without the full ~175 GB of experts. Unset in normal use.
|
|
_cap = os.environ.get("FREETOKEN_GLM5_MAX_LAYERS")
|
|
if _cap:
|
|
num_layers = min(num_layers, int(_cap))
|
|
|
|
kda_ids = tuple(i for i in args.kda_layer_ids if i < num_layers)
|
|
dsa_ids = tuple(i for i in args.dsa_layer_ids if i < num_layers)
|
|
|
|
# NoPE: the main attention carries no rotary dims (rotary_dim == 0); rope
|
|
# survives only in the indexer geometry (args.rope_theta / interleave).
|
|
rotary_config = RotaryConfig(
|
|
head_dim=args.qk_head_dim,
|
|
rotary_dim=args.qk_rope_head_dim,
|
|
max_position=args.max_position,
|
|
base=args.rope_theta,
|
|
scaling=None,
|
|
)
|
|
latent_dim = args.latent_dim # 512: bare ckv, no kpe rows
|
|
|
|
dsa_on = _dsa_on(args, dsa_ids)
|
|
# Each DSA layer owns its indexer ("full" in indexer_types); count only the
|
|
# layers that actually exist under a dev layer cap.
|
|
num_index_layers = (
|
|
sum(
|
|
1
|
|
for i in dsa_ids
|
|
if i < len(args.indexer_types) and args.indexer_types[i] == "full"
|
|
)
|
|
if dsa_on
|
|
else 0
|
|
)
|
|
|
|
linear_group = LinearGatedDeltaGroupConfig(
|
|
name="linear",
|
|
layer_ids=kda_ids,
|
|
# KDA: one qk-sized and one v-sized head set (H=64, d=128 each). Mapping onto
|
|
# the GDN field names keeps LinearStatePool's conv-width formula exact:
|
|
# 2*K*dk + V*dv = (q|k) + v = 3 * H * d.
|
|
num_key_heads=args.linear_num_heads,
|
|
num_value_heads=args.linear_num_heads,
|
|
key_head_dim=args.linear_head_dim,
|
|
value_head_dim=args.linear_head_dim,
|
|
conv_kernel_dim=args.linear_conv_kernel_dim,
|
|
output_gate="sigmoid", # KDA o_norm gates with sigmoid (kda.py)
|
|
variant="kda",
|
|
)
|
|
full_group = FullAttentionGroupConfig(
|
|
name="full",
|
|
layer_ids=dsa_ids,
|
|
num_kv_heads=1, # single shared MLA latent
|
|
head_dim=latent_dim,
|
|
rotary_config=rotary_config,
|
|
mla=True,
|
|
index_head_dim=args.index_head_dim if dsa_on else 0,
|
|
num_index_layers=num_index_layers,
|
|
index_ratio=(args.index_kpool if dsa_on and args.index_kpool_compress else 1),
|
|
)
|
|
groups = tuple(
|
|
sorted(
|
|
(linear_group, full_group),
|
|
key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,
|
|
)
|
|
)
|
|
|
|
# The MLP layout is a dense prefix + sparse tail; ModelConfig models exactly that
|
|
# via first_k_dense_replace, so assert the checkpoint matches before collapsing.
|
|
mlp_types = args.mlp_layer_types[:num_layers]
|
|
first_dense = next(
|
|
(i for i, t in enumerate(mlp_types) if t == "sparse"), len(mlp_types)
|
|
)
|
|
assert all(t == "dense" for t in mlp_types[:first_dense]) and all(
|
|
t == "sparse" for t in mlp_types[first_dense:]
|
|
), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"
|
|
|
|
expert_quant, weight_block_size = _fp8_block_quant(hf_config)
|
|
if expert_quant == "none":
|
|
expert_quant = detect_expert_quant(hf_config)
|
|
return ModelConfig(
|
|
num_layers=num_layers,
|
|
num_qo_heads=args.num_heads,
|
|
num_kv_heads=1,
|
|
head_dim=latent_dim,
|
|
hidden_size=args.hidden_size,
|
|
vocab_size=text.vocab_size,
|
|
intermediate_size=text.intermediate_size,
|
|
# hidden_act stands proxy for the EXPERT activation everywhere the engine
|
|
# gates on it (NVFP4 backend selection, CPU-executor capability): GLM-5.3
|
|
# experts and dense MLPs both run clamped SwiGLU when swiglu_limit is set,
|
|
# even though the HF config still says "silu". Passing "silu" through would
|
|
# let auto-selection repack experts into the marlin/b12x silu-only epilogue.
|
|
hidden_act=(
|
|
"swiglu_clamp" if args.swiglu_limit is not None else text.hidden_act
|
|
),
|
|
rms_norm_eps=args.norm_eps,
|
|
tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
|
|
rotary_config=rotary_config,
|
|
attention_groups=groups,
|
|
num_experts=(
|
|
getattr(text, "n_routed_experts", None) or getattr(text, "num_experts", 0)
|
|
),
|
|
num_experts_per_tok=(
|
|
getattr(text, "num_experts_per_tok", None)
|
|
or getattr(text, "num_experts_per_token", 0)
|
|
),
|
|
moe_intermediate_size=getattr(text, "moe_intermediate_size", 0)
|
|
or text.intermediate_size,
|
|
norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),
|
|
model_type=getattr(hf_config, "model_type", "glm5_next"),
|
|
architectures=getattr(
|
|
hf_config, "architectures", ["Glm5NextForConditionalGeneration"]
|
|
),
|
|
moe_enabled=True,
|
|
expert_quant=expert_quant,
|
|
weight_block_size=weight_block_size,
|
|
first_k_dense_replace=first_dense,
|
|
n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),
|
|
routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
|
|
n_group=int(getattr(text, "n_group", 1) or 1),
|
|
topk_group=int(getattr(text, "topk_group", 1) or 1),
|
|
attn_sm_scale=args.qk_head_dim**-0.5,
|
|
has_attn_bias=bool(getattr(text, "attention_bias", False)),
|
|
swiglu_limit=args.swiglu_limit,
|
|
vision_config=parse_vision_config(hf_config),
|
|
image_token_id=getattr(hf_config, "image_token_id", None),
|
|
glm5_args=args,
|
|
)
|
|
|
|
|
|
__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]
|