mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* fix(qwen4_exp): read weights under the checkpoint's quant config - route each dense Linear tensor by its scheme's stored names, so per-channel fp8 scales (llm-compressor) load and fused parts concatenate per role - name the routed NVFP4 experts by the checkpoint's dialect, so compressed-tensors weight_packed / weight_global_scale load - accept a bf16 PLE table without a scale in the pinned and disk backends * fix(quant): accept weight_scale as the modelopt mxfp8 scale name ModelOpt exports store the same e8m0 codes as weight_scale_inv (MiniMax-M3) or weight_scale (Qwen3.8-Flash-Next-FP4-Local); Stored gains aliases and the scheme-driven readers match any of them.
140 lines
5.6 KiB
Python
140 lines
5.6 KiB
Python
"""QuantConfig: reads one quantization_config dialect and answers what scheme each module has."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from typing import Any, ClassVar
|
|
|
|
from ..embedding import EmbeddingConfig
|
|
from ..linear import LinearConfig
|
|
from ..moe import MoEConfig
|
|
from ..names import Matcher, NameMap, name_set
|
|
from ..quant_backend import get_quant_backend
|
|
from ..registry import LayerKind, dialects, method_class, methods_for, register_dialect
|
|
from ..scheme import QuantKind, QuantScheme
|
|
|
|
LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig, LayerKind.EMBEDDING: EmbeddingConfig}
|
|
|
|
|
|
def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
|
|
if cfg is None:
|
|
return default
|
|
if isinstance(cfg, dict):
|
|
return cfg.get(key, default)
|
|
return getattr(cfg, key, default)
|
|
|
|
|
|
def quantization_config_of(hf_config: Any) -> dict[str, Any] | None:
|
|
q = cfg_get(hf_config, "quantization_config")
|
|
if q is None:
|
|
q = cfg_get(cfg_get(hf_config, "text_config"), "quantization_config")
|
|
if q is None:
|
|
return None
|
|
if isinstance(q, dict):
|
|
return q
|
|
if hasattr(q, "to_dict"):
|
|
return dict(q.to_dict())
|
|
return dict(vars(q))
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Stored:
|
|
"""One checkpoint tensor behind a role: its suffix, other suffixes exports of the same dialect use for it, and whether it holds the quant-side scale whose reciprocal the layer wants."""
|
|
|
|
name: str
|
|
reciprocal: bool = False
|
|
aliases: tuple[str, ...] = ()
|
|
|
|
@property
|
|
def names(self) -> tuple[str, ...]:
|
|
return (self.name, *self.aliases)
|
|
|
|
|
|
class QuantConfig(ABC):
|
|
dialect: ClassVar[str]
|
|
# per kind the dialect exports, role -> the checkpoint tensor suffix (or Stored) that feeds it
|
|
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]]
|
|
|
|
def __init__(self, name_map: NameMap | None = None, unquantized: tuple[str, ...] = ()):
|
|
self.name_map = name_map or NameMap()
|
|
# modules the family serves in bf16 regardless of the dialect (glob patterns on checkpoint names)
|
|
self.unquantized: Matcher = name_set(tuple(unquantized))
|
|
self._schemes: dict[str, QuantScheme | None] = {}
|
|
|
|
@classmethod
|
|
def claims(cls, q: dict[str, Any]) -> bool:
|
|
"""Whether this dialect reads the given ``quantization_config``."""
|
|
return str(q.get("quant_method") or "").lower() == cls.dialect
|
|
|
|
@abstractmethod
|
|
def scheme_for_name(self, name: str) -> QuantScheme | None:
|
|
"""Scheme of one module by its checkpoint name; None means unquantized."""
|
|
|
|
def scheme_for(self, prefix: str) -> QuantScheme | None:
|
|
if prefix in self._schemes:
|
|
return self._schemes[prefix]
|
|
names = self.name_map.to_checkpoint(prefix)
|
|
schemes = {None if self.unquantized(n) else self.scheme_for_name(n) for n in names}
|
|
if len(schemes) != 1:
|
|
raise ValueError(f"fused module {prefix!r} mixes quantization schemes across {names}: {schemes}")
|
|
scheme = schemes.pop()
|
|
self._schemes[prefix] = scheme
|
|
return scheme
|
|
|
|
def stored_tensors(self, kind: QuantKind) -> dict[str, Stored]:
|
|
"""role -> checkpoint tensor for every role the dialect stores for ``kind``."""
|
|
return {role: entry if isinstance(entry, Stored) else Stored(entry) for role, entry in self.STORAGE[kind].items()}
|
|
|
|
def storage(self, scheme: QuantScheme) -> dict[str, Stored]:
|
|
"""role -> checkpoint tensor for one scheme's tensors."""
|
|
return {role: entry for role, entry in self.stored_tensors(scheme.kind).items() if scheme.has(role)}
|
|
|
|
def get_quant_method(self, layer: Any, prefix: str):
|
|
scheme = self.scheme_for(prefix)
|
|
layer_kind = layer.quant_layer_kind
|
|
kind = scheme.kind if scheme else QuantKind.NONE
|
|
if layer_kind is LayerKind.EMBEDDING and kind not in methods_for(layer_kind):
|
|
# a checkpoint that quantizes its linears stores the vocab table dense
|
|
scheme, kind = None, QuantKind.NONE
|
|
cls = method_class(kind, layer_kind)
|
|
cfg = self.layer_config(layer, layer_kind, scheme)
|
|
return cls(cfg, get_quant_backend().select(layer_kind, kind))
|
|
|
|
def layer_config(self, layer: Any, layer_kind: LayerKind, scheme: QuantScheme | None) -> LinearConfig | MoEConfig:
|
|
"""The layer's Linear / MoE config; a dialect overrides it to add what only its checkpoint knows."""
|
|
return LAYER_CONFIGS[layer_kind].from_layer(layer, scheme)
|
|
|
|
@staticmethod
|
|
def from_hf(
|
|
hf_config: Any,
|
|
*,
|
|
name_map: NameMap | None = None,
|
|
unquantized: tuple[str, ...] = (),
|
|
) -> "QuantConfig":
|
|
"""``unquantized`` lists the modules the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors).
|
|
|
|
``hf_config`` comes from ``cached_load_hf_config``, which folds an old ModelOpt ``hf_quant_config.json`` into ``quantization_config``."""
|
|
q = quantization_config_of(hf_config)
|
|
if q is None:
|
|
return NoQuantConfig(name_map, unquantized)
|
|
for cls in dialects():
|
|
if cls.claims(q):
|
|
return cls(q, hf_config, name_map=name_map, unquantized=unquantized)
|
|
raise NotImplementedError(f"quantization method {q.get('quant_method')!r} is not supported")
|
|
|
|
|
|
@register_dialect
|
|
class NoQuantConfig(QuantConfig):
|
|
"""No ``quantization_config``: every module is bf16."""
|
|
|
|
dialect = "none"
|
|
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {}
|
|
|
|
@classmethod
|
|
def claims(cls, q: dict[str, Any]) -> bool:
|
|
return False
|
|
|
|
def scheme_for_name(self, name: str) -> QuantScheme | None:
|
|
return None
|