mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
82 lines
4.1 KiB
Python
82 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any, ClassVar
|
|
|
|
from ..names import ancestors, name_set
|
|
from ..registry import register_dialect
|
|
from ..scheme import QuantKind, QuantScheme
|
|
from ..scheme import fp8_block_scheme, fp8_tensor_scheme, mxfp8_scheme, nvfp4_scheme
|
|
from .base import QuantConfig, Stored
|
|
|
|
|
|
@register_dialect
|
|
class ModelOptConfig(QuantConfig):
|
|
"""NVIDIA ModelOpt exports: one ``quant_algo`` for every Linear minus ``ignore``, or
|
|
``MIXED_PRECISION`` with a per-module ``quantized_layers`` allow-list."""
|
|
|
|
dialect = "modelopt"
|
|
|
|
SCHEMES: ClassVar[dict[str, QuantScheme]] = {
|
|
"NVFP4": nvfp4_scheme(input_scale=True),
|
|
"W4A16_NVFP4": nvfp4_scheme(input_scale=False),
|
|
"FP8": fp8_tensor_scheme("fp32", input_scale=True),
|
|
"FP8_PER_CHANNEL_PER_TOKEN": fp8_tensor_scheme("fp32", per_row=True),
|
|
"FP8_PB_WO": fp8_block_scheme("fp32"),
|
|
"MXFP8": mxfp8_scheme(),
|
|
}
|
|
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {
|
|
QuantKind.FP8_TENSOR: {"weight": "weight", "weight_scale": "weight_scale", "input_scale": "input_scale"},
|
|
QuantKind.FP8_BLOCK: {"weight": "weight", "weight_scale_inv": "weight_scale_inv"},
|
|
# same e8m0 codes under either name: MiniMax-M3 exports weight_scale_inv, Qwen3.8-Flash-Next weight_scale
|
|
QuantKind.MXFP8: {"weight": "weight", "weight_scale_inv": Stored("weight_scale_inv", aliases=("weight_scale",))},
|
|
QuantKind.NVFP4: {"weight": "weight", "weight_scale": "weight_scale", "weight_global": "weight_scale_2", "input_scale": "input_scale"},
|
|
}
|
|
|
|
@classmethod
|
|
def claims(cls, q: dict[str, Any]) -> bool:
|
|
method = str(q.get("quant_method") or "").lower()
|
|
return method in ("modelopt", "modelopt_mixed") or (not method and bool(q.get("quant_algo")))
|
|
|
|
def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
|
|
super().__init__(name_map, unquantized)
|
|
self.algo = str(q.get("quant_algo") or "").upper()
|
|
# Inferact exports carry this key; false says the FP8 and NVFP4 layers store no input_scale, and an explicit value wins over the config_groups rule below
|
|
self.with_input_scale = q.get("with_input_scale")
|
|
groups = q.get("config_groups")
|
|
# an export with no activation quantizer can still say NVFP4; every config group then has input_activations null (vLLM applies the same rule)
|
|
if self.with_input_scale is None and self.algo == "NVFP4" and isinstance(groups, dict) and groups and all(isinstance(g, dict) and g.get("input_activations") is None for g in groups.values()):
|
|
self.algo = "W4A16_NVFP4"
|
|
self.ignore = name_set(tuple(q.get("ignore") or q.get("exclude_modules") or ()))
|
|
layers = q.get("quantized_layers") or {}
|
|
self.quantized_layers = {k: str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items()} if isinstance(layers, dict) else {}
|
|
if self.algo == "MIXED_PRECISION" and not self.quantized_layers:
|
|
raise NotImplementedError("ModelOpt MIXED_PRECISION without quantized_layers in quantization_config")
|
|
if self.algo != "MIXED_PRECISION":
|
|
self._scheme_of(self.algo)
|
|
|
|
def scheme_for_name(self, name: str) -> QuantScheme | None:
|
|
if self.ignore(name):
|
|
return None
|
|
algo = self._module_algo(name)
|
|
return None if algo is None else self._scheme_of(algo)
|
|
|
|
def _module_algo(self, name: str) -> str | None:
|
|
if self.algo != "MIXED_PRECISION":
|
|
return self.algo
|
|
for a in ancestors(name):
|
|
algo = self.quantized_layers.get(a)
|
|
if algo is not None:
|
|
return algo
|
|
return None
|
|
|
|
def _scheme_of(self, algo: str) -> QuantScheme:
|
|
if self.with_input_scale is False:
|
|
if algo == "FP8":
|
|
return fp8_tensor_scheme("fp32") # ModelOpt has no algo name for fp8 without an activation scale
|
|
if algo == "NVFP4":
|
|
algo = "W4A16_NVFP4"
|
|
try:
|
|
return self.SCHEMES[algo]
|
|
except KeyError:
|
|
raise NotImplementedError(f"ModelOpt quant_algo {algo!r} is not supported") from None
|