Files
Xiaoze Fan 477c8601b2 refactor(quant): config, scheme and method layers for quantization (#418)
* refactor(quant): config, scheme and method layers for checkpoint quantization

QuantConfig parses the checkpoint's quantization_config into a QuantScheme per
projection; every Linear*, ParallelLMHead and MoE layer takes quant_config and a
prefix, declares its tensors through a QuantMethod and selects its kernel at
construction. MoE kernels own the expert bank layout, pack and apply, so the
offload cache, the FTW converter and the CPU executor follow one layout.

- --quant-backend layer[.kind]=name selects kernels (--nvfp4-backend forwards to it)
- --moe-strategy replaces --moe-backend; the old spelling stays as an alias
- --moe-cpu-layers auto for Windows/WSL pin budgets; --moe-cache-auto respects the kernel's slot limit
- removes the per-family quantized layer classes, bank providers, format env switches and GLM runtime fp8
2026-09-08 23:37:23 -07:00

86 lines
3.0 KiB
Python

from __future__ import annotations
from typing import TYPE_CHECKING
from freetoken.core import get_global_ctx
from freetoken.distributed import get_tp_info
from freetoken.layers import BaseOP, LinearOProj, LinearQKVMerged, RMSNorm
from freetoken.layers.rotary import get_rope
from freetoken.utils import div_even, nvtx_annotate
if TYPE_CHECKING:
import torch
from freetoken.models.config import ModelConfig
class LlamaAttention(BaseOP):
def __init__(
self,
config: ModelConfig,
layer_id: int,
*,
has_attn_bias: bool = False,
has_qk_norm: bool = False,
prefix: str = "",
):
head_dim = config.head_dim
self.layer_id = layer_id
tp_size = get_tp_info().size
self.num_qo_heads = div_even(config.num_qo_heads, tp_size)
self.num_kv_heads = div_even(config.num_kv_heads, tp_size, allow_replicate=True)
self.qo_attn_dim = self.num_qo_heads * head_dim
self.kv_attn_dim = self.num_kv_heads * head_dim
self.head_dim = head_dim
self.qkv_proj = LinearQKVMerged(
hidden_size=config.hidden_size,
head_dim=config.head_dim,
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=has_attn_bias,
quant_config=config.quant,
prefix=f"{prefix}.qkv_proj",
)
if has_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
else:
self.q_norm = None
self.k_norm = None
self.rotary = get_rope(
head_dim=head_dim,
rotary_dim=config.rotary_config.rotary_dim,
max_position=config.rotary_config.max_position,
base=config.rotary_config.base,
rope_scaling=(
tuple(config.rotary_config.scaling.items())
if config.rotary_config.scaling
else None
),
)
self.o_proj = LinearOProj(
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
quant_config=config.quant,
prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
def forward(self, x: torch.Tensor) -> torch.Tensor:
ctx = get_global_ctx()
qkv = self.qkv_proj.forward(x)
del x
q, k, v = qkv.split([self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim], dim=-1)
if self.q_norm is not None:
self.q_norm.forward_inplace(q.view(-1, self.num_qo_heads, self.head_dim))
if self.k_norm is not None:
self.k_norm.forward_inplace(k.view(-1, self.num_kv_heads, self.head_dim))
q, k = self.rotary.forward(ctx.batch.positions, q, k)
q = q.view(-1, self.num_qo_heads, self.head_dim)
o = ctx.attn_backend.forward(q, k, v, self.layer_id, ctx.batch)
return self.o_proj.forward(o.view(-1, self.qo_attn_dim))
__all__ = ["LlamaAttention"]