mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor(quant): config, scheme and method layers for checkpoint quantization QuantConfig parses the checkpoint's quantization_config into a QuantScheme per projection; every Linear*, ParallelLMHead and MoE layer takes quant_config and a prefix, declares its tensors through a QuantMethod and selects its kernel at construction. MoE kernels own the expert bank layout, pack and apply, so the offload cache, the FTW converter and the CPU executor follow one layout. - --quant-backend layer[.kind]=name selects kernels (--nvfp4-backend forwards to it) - --moe-strategy replaces --moe-backend; the old spelling stays as an alias - --moe-cpu-layers auto for Windows/WSL pin budgets; --moe-cache-auto respects the kernel's slot limit - removes the per-family quantized layer classes, bank providers, format env switches and GLM runtime fp8
49 lines
1.7 KiB
Python
49 lines
1.7 KiB
Python
"""fp8 e4m3 weight with e8m0 scales per 32 input elements (OCP MXFP8)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import MX_GROUP as BLOCK, QuantKind
|
|
from .base import LinearKernel, LinearMethod
|
|
|
|
FP8 = torch.float8_e4m3fn
|
|
|
|
|
|
class TritonMxfp8LinearKernel(LinearKernel):
|
|
"""W8A16: split-K GEMV up to M=256, bf16 dequant + cuBLAS above."""
|
|
|
|
name = "triton"
|
|
|
|
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel.triton.mxfp8_linear import mxfp8_linear
|
|
|
|
return mxfp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)
|
|
|
|
|
|
class EmulationMxfp8LinearKernel(LinearKernel):
|
|
name = "emulation"
|
|
|
|
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant
|
|
|
|
*lead, K = x.shape
|
|
w = mxfp8_dequant(layer.weight, layer.weight_scale_inv, dtype=torch.float32)
|
|
out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])
|
|
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
|
|
|
|
|
|
@register_method(QuantKind.MXFP8, LayerKind.LINEAR)
|
|
class Mxfp8LinearMethod(LinearMethod):
|
|
candidates = (TritonMxfp8LinearKernel, EmulationMxfp8LinearKernel)
|
|
|
|
def create_weights(self, layer: Any) -> None:
|
|
g = self.cfg
|
|
if g.in_features % BLOCK:
|
|
raise ValueError(f"mxfp8 needs in_features divisible by {BLOCK}, got {g.in_features}")
|
|
layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
|
|
layer.weight_scale_inv = torch.empty(g.out_features, g.in_features // BLOCK, dtype=torch.uint8)
|