Files
Xiaoze Fan 56e7f3ae38 feat(quant)!: compress bf16 weights to df11 at load with --online-quant (#891)
* feat(quant)!: compress bf16 weights to df11 at load with --online-quant

GLM-4.7 no longer hardwires DF11 attention and embedding; it serves them
bf16 unless --online-quant df11 is passed, and FTW files converted with the
old DF11 tensors must be reconverted.

* fix(quant): skip layers --online-quant cannot convert instead of failing
2026-11-11 13:24:46 -08:00

83 lines
2.8 KiB
Python

"""QuantMethod: one (kind, layer kind) pair bound to one kernel object; the selector and the post-load hook."""
from __future__ import annotations
from abc import ABC
from typing import Any, ClassVar, Sequence
from freetoken.utils import init_logger
from .registry import LayerKind
from .scheme import QuantKind
logger = init_logger(__name__)
class KernelSelectionError(RuntimeError):
pass
def select_kernel(candidates: Sequence[type], requested: str, cfg: Any):
"""Pick one kernel from an ordered table: the requested name if usable, else the first usable one that is worth it."""
if not candidates:
raise KernelSelectionError("empty kernel table")
names = [c.name for c in candidates]
if requested != "auto" and requested in names:
kernel = candidates[names.index(requested)]()
reason = kernel.unusable_reason(cfg)
if reason:
raise KernelSelectionError(f"kernel {requested!r} was requested but cannot run here: {reason}")
return kernel
skipped: list[str] = []
fallback = None
for cls in candidates:
kernel = cls()
reason = kernel.unusable_reason(cfg)
if reason:
skipped.append(f"{kernel.name}: {reason}")
continue
if kernel.worth_it(cfg):
if skipped:
# every layer selects its own kernel
logger.info_once("kernel %s selected; skipped %s", kernel.name, "; ".join(skipped))
return kernel
if fallback is None:
fallback = kernel
if fallback is not None:
return fallback
raise KernelSelectionError("no usable kernel in table; " + "; ".join(skipped))
class QuantMethod(ABC):
kind: ClassVar[QuantKind]
layer_kind: ClassVar[LayerKind]
candidates: ClassVar[tuple[type, ...]] = ()
# declares the bf16 weight the checkpoint stores and converts it in finalize; the engine loads such weights to host first
online: ClassVar[bool] = False
def __init__(self, cfg: Any, requested: str = "auto"):
self.cfg = cfg
self.requested = requested
self.kernel = select_kernel(self.candidates, requested, cfg)
@classmethod
def refuses(cls, layer: Any) -> str | None:
"""Why this online method leaves ``layer`` as the checkpoint stores it, or None when it converts it."""
return None
@property
def scheme(self):
return self.cfg.scheme
def finalize_quant(root: Any) -> int:
"""Call ``quant_method.finalize(layer)`` on every layer under ``root``; returns the count."""
from freetoken.layers.base import iter_ops
count = 0
for op in iter_ops(root):
method = getattr(op, "quant_method", None)
if method is not None:
method.finalize(op)
count += 1
return count