mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
debug_once / info_once / warning_once log on rank 0, once per distinct message and args. select_kernel, which logged its choice once per layer, and the hand-rolled log-once guards now use them.
76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
"""QuantMethod: one (kind, layer kind) pair bound to one kernel object; the selector and the post-load hook."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC
|
|
from typing import Any, ClassVar, Sequence
|
|
|
|
from freetoken.utils import init_logger
|
|
|
|
from .registry import LayerKind
|
|
from .scheme import QuantKind
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class KernelSelectionError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def select_kernel(candidates: Sequence[type], requested: str, cfg: Any):
|
|
"""Pick one kernel from an ordered table: the requested name if usable, else the first usable one that is worth it."""
|
|
if not candidates:
|
|
raise KernelSelectionError("empty kernel table")
|
|
names = [c.name for c in candidates]
|
|
if requested != "auto" and requested in names:
|
|
kernel = candidates[names.index(requested)]()
|
|
reason = kernel.unusable_reason(cfg)
|
|
if reason:
|
|
raise KernelSelectionError(f"kernel {requested!r} was requested but cannot run here: {reason}")
|
|
return kernel
|
|
skipped: list[str] = []
|
|
fallback = None
|
|
for cls in candidates:
|
|
kernel = cls()
|
|
reason = kernel.unusable_reason(cfg)
|
|
if reason:
|
|
skipped.append(f"{kernel.name}: {reason}")
|
|
continue
|
|
if kernel.worth_it(cfg):
|
|
if skipped:
|
|
# every layer selects its own kernel
|
|
logger.info_once("kernel %s selected; skipped %s", kernel.name, "; ".join(skipped))
|
|
return kernel
|
|
if fallback is None:
|
|
fallback = kernel
|
|
if fallback is not None:
|
|
return fallback
|
|
raise KernelSelectionError("no usable kernel in table; " + "; ".join(skipped))
|
|
|
|
|
|
class QuantMethod(ABC):
|
|
kind: ClassVar[QuantKind]
|
|
layer_kind: ClassVar[LayerKind]
|
|
candidates: ClassVar[tuple[type, ...]] = ()
|
|
|
|
def __init__(self, cfg: Any, requested: str = "auto"):
|
|
self.cfg = cfg
|
|
self.requested = requested
|
|
self.kernel = select_kernel(self.candidates, requested, cfg)
|
|
|
|
@property
|
|
def scheme(self):
|
|
return self.cfg.scheme
|
|
|
|
def finalize_quant(root: Any) -> int:
|
|
"""Call ``quant_method.finalize(layer)`` on every layer under ``root``; returns the count."""
|
|
from freetoken.layers.base import iter_ops
|
|
|
|
count = 0
|
|
for op in iter_ops(root):
|
|
method = getattr(op, "quant_method", None)
|
|
if method is not None:
|
|
method.finalize(op)
|
|
count += 1
|
|
return count
|