Files
Xiaoze Fan 56e7f3ae38 feat(quant)!: compress bf16 weights to df11 at load with --online-quant (#891)
* feat(quant)!: compress bf16 weights to df11 at load with --online-quant

GLM-4.7 no longer hardwires DF11 attention and embedding; it serves them
bf16 unless --online-quant df11 is passed, and FTW files converted with the
old DF11 tensors must be reconverted.

* fix(quant): skip layers --online-quant cannot convert instead of failing
2026-11-11 13:24:46 -08:00

79 lines
2.9 KiB
Python

"""DF11 Linear: the bf16 weight compressed losslessly at load, decoded bit-for-bit into a shared scratch before each GEMM."""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import torch
from ..registry import LayerKind, register_method
from ..scheme import QuantKind
from .base import LinearConfig, LinearKernel, LinearMethod
from .unquantized import TorchLinearKernel
DF11_KEYS = ("low8", "bitstream", "chunk_start", "lut")
# decodes run one layer at a time, so every DF11 linear on a device shares one scratch sized to the largest weight
_SCRATCH: dict[torch.device, torch.Tensor] = {}
_GEMM = TorchLinearKernel()
def _scratch(n: int, device: torch.device) -> torch.Tensor:
buf = _SCRATCH.get(device)
if buf is None or buf.numel() < n:
buf = torch.empty(n, dtype=torch.int16, device=device)
_SCRATCH[device] = buf
return buf[:n]
def df11_install(layer: Any, compressed: dict) -> None:
"""Replace ``layer.weight`` with the DF11 tensors of ``compressed``."""
del layer.weight
for key in DF11_KEYS:
setattr(layer, key, compressed[key])
layer.df11_meta = compressed["meta"]
def df11_bundle(layer: Any) -> dict:
return {**{key: getattr(layer, key) for key in DF11_KEYS}, "meta": layer.df11_meta}
class Df11LinearKernel(LinearKernel):
name = "triton"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
# the model is built under its dtype, which the loader casts the checkpoint weight to
dtype = torch.get_default_dtype()
return None if dtype == torch.bfloat16 else f"DF11 compresses bf16 weights, the model runs {dtype}"
def finalize(self, layer: Any) -> None:
from freetoken.kernel.triton.df11 import df11_compress
df11_install(layer, df11_compress(layer.weight.cuda()))
# sized here, not in the first forward, so a CUDA graph capture never grows it
_scratch(layer.out_features * layer.in_features, layer.low8.device)
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.triton.df11_decode import df11_decompress
out = _scratch(layer.out_features * layer.in_features, x.device).view(layer.out_features, layer.in_features)
return _GEMM.apply(SimpleNamespace(weight=df11_decompress(df11_bundle(layer), out=out), bias=layer.bias), x)
@register_method(QuantKind.DF11, LayerKind.LINEAR)
class Df11LinearMethod(LinearMethod):
candidates = (Df11LinearKernel,)
online = True
@classmethod
def refuses(cls, layer: Any) -> str | None:
from freetoken.layers.embedding import ParallelLMHead
if isinstance(layer, ParallelLMHead):
return "the LM head would decode its whole weight every step"
return None
def create_weights(self, layer: Any) -> None:
layer.weight = torch.empty(self.cfg.out_features, self.cfg.in_features)