mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor(quant): config, scheme and method layers for checkpoint quantization QuantConfig parses the checkpoint's quantization_config into a QuantScheme per projection; every Linear*, ParallelLMHead and MoE layer takes quant_config and a prefix, declares its tensors through a QuantMethod and selects its kernel at construction. MoE kernels own the expert bank layout, pack and apply, so the offload cache, the FTW converter and the CPU executor follow one layout. - --quant-backend layer[.kind]=name selects kernels (--nvfp4-backend forwards to it) - --moe-strategy replaces --moe-backend; the old spelling stays as an alias - --moe-cpu-layers auto for Windows/WSL pin budgets; --moe-cache-auto respects the kernel's slot limit - removes the per-family quantized layer classes, bank providers, format env switches and GLM runtime fp8
81 lines
2.8 KiB
Python
81 lines
2.8 KiB
Python
"""The --quant-backend value: the kernel name wanted for each table, ``layer=name`` for every kind of a layer type, ``layer.kind=name`` for one."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from .registry import LayerKind, method_class, methods_for
|
|
from .scheme import QuantKind
|
|
|
|
Table = tuple[LayerKind, "QuantKind | None"]
|
|
|
|
|
|
def _kernel_names(layer_kind: LayerKind, kind: QuantKind | None) -> set[str]:
|
|
if kind is not None:
|
|
try:
|
|
return {c.name for c in method_class(kind, layer_kind).candidates}
|
|
except NotImplementedError:
|
|
return set()
|
|
return {c.name for cls in methods_for(layer_kind).values() for c in cls.candidates}
|
|
|
|
|
|
def _parse_table(key: str) -> Table | None:
|
|
layer, _, kind = key.partition(".")
|
|
try:
|
|
return LayerKind(layer.strip()), QuantKind(kind.strip()) if kind else None
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _table_text(table: Table) -> str:
|
|
layer, kind = table
|
|
return f"{layer}.{kind}" if kind is not None else str(layer)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class QuantBackend:
|
|
items: tuple[tuple[Table, str], ...] = ()
|
|
|
|
@classmethod
|
|
def parse(cls, text: str | None) -> "QuantBackend":
|
|
items: dict[Table, str] = {}
|
|
for raw in (text or "").split(","):
|
|
item = raw.strip()
|
|
if not item:
|
|
continue
|
|
key, sep, name = item.partition("=")
|
|
table = _parse_table(key)
|
|
name = name.strip().lower()
|
|
if not sep or not name or table is None:
|
|
layers = [l.value for l in LayerKind]
|
|
raise ValueError(f"bad --quant-backend item {item!r}; expected layer[.kind]=name with layer in {layers}")
|
|
known = _kernel_names(*table)
|
|
if name not in known:
|
|
raise ValueError(f"--quant-backend {key}: no kernel {name!r}; known: {sorted(known)}")
|
|
items[table] = name
|
|
return cls(tuple(sorted(items.items(), key=lambda kv: _table_text(kv[0]))))
|
|
|
|
def select(self, layer_kind: LayerKind, kind: QuantKind) -> str:
|
|
"""The kernel name for one table: its own entry, else the layer-wide entry when the table lists that kernel, else ``"auto"``."""
|
|
entries = dict(self.items)
|
|
exact = entries.get((layer_kind, kind))
|
|
if exact:
|
|
return exact
|
|
broad = entries.get((layer_kind, None))
|
|
if broad and broad in _kernel_names(layer_kind, kind):
|
|
return broad
|
|
return "auto"
|
|
|
|
|
|
_QUANT_BACKEND = QuantBackend()
|
|
|
|
|
|
def set_quant_backend(backend: QuantBackend) -> None:
|
|
"""Install the kernel requests every layer built afterwards consults; the engine sets them once before create_model."""
|
|
global _QUANT_BACKEND
|
|
_QUANT_BACKEND = backend
|
|
|
|
|
|
def get_quant_backend() -> QuantBackend:
|
|
return _QUANT_BACKEND
|