mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 07:03:05 +08:00
* feat(kvcache): add fp8 and int8 per-token-head KV cache formats * fix(kernel): keep the bf16 m3 sparse k/v loads inline * fix(attention): refuse a backend that cannot read the kv format - filter the m3 page-size fallback by kv format - release the scale buffer when a qsa or bsa rebuild fails - keep the bf16 qsa and m3 attend code identical to main - drop docstrings that claimed no register spills - drop guard tests * fix(kernel): widen dsa latent row offsets past int32 * feat(spec): store the mtp draft kv in the target's format
92 lines
3.9 KiB
Python
92 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import torch
|
|
|
|
ROW = -1 # one scale per row
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class KVFormat:
|
|
"""How one K or V row is stored in the KV pool; the encoding lives in the ``kv_rows.py`` arms of the same name."""
|
|
|
|
name: str # --kv-cache-dtype value and the kernels' KV_FMT
|
|
elem_dtype: torch.dtype | None # logical element type; None follows the compute dtype
|
|
elem_store_dtype: torch.dtype | None # dtype of the element tensor; None follows the compute dtype
|
|
scale_dtype: torch.dtype | None = None # logical scale type; None means no scale tensor
|
|
scale_store_dtype: torch.dtype | None = None # dtype of the scale tensor
|
|
scale_group: int = ROW # channels per scale
|
|
|
|
def __post_init__(self) -> None:
|
|
if (self.elem_dtype is None) != (self.elem_store_dtype is None):
|
|
raise ValueError(f"{self.name}: elem_dtype and elem_store_dtype must be set together")
|
|
if (self.scale_dtype is None) != (self.scale_store_dtype is None):
|
|
raise ValueError(f"{self.name}: scale_dtype and scale_store_dtype must be set together")
|
|
if self.scale_dtype is not None and self.scale_dtype.itemsize != self.scale_store_dtype.itemsize:
|
|
raise ValueError(f"{self.name}: scale_store_dtype must have the size of scale_dtype")
|
|
|
|
@property
|
|
def quantized(self) -> bool:
|
|
return self.elem_dtype is not None
|
|
|
|
def elem_tensor_dtype(self, compute_dtype: torch.dtype) -> torch.dtype:
|
|
return compute_dtype if self.elem_store_dtype is None else self.elem_store_dtype
|
|
|
|
def elem_dim(self, head_dim: int) -> int:
|
|
"""Elements of the element tensor per row."""
|
|
if self.elem_dtype is None:
|
|
return head_dim
|
|
row_bytes = head_dim * self.elem_dtype.itemsize
|
|
if row_bytes % self.elem_store_dtype.itemsize:
|
|
raise ValueError(f"{self.name} cannot store a {head_dim}-wide row")
|
|
return row_bytes // self.elem_store_dtype.itemsize
|
|
|
|
def scale_dim(self, head_dim: int) -> int:
|
|
"""Scales per row."""
|
|
if self.scale_dtype is None:
|
|
return 0
|
|
if self.scale_group == ROW:
|
|
return 1
|
|
if head_dim % self.scale_group:
|
|
raise ValueError(f"{self.name} needs head_dim divisible by {self.scale_group}, got {head_dim}")
|
|
return head_dim // self.scale_group
|
|
|
|
def row_bytes(self, head_dim: int, compute_dtype: torch.dtype, segments: int = 1) -> int:
|
|
"""Bytes of one K or V row split into ``segments`` separately scaled parts, its scales included."""
|
|
elem_bytes = self.elem_dim(head_dim) * self.elem_tensor_dtype(compute_dtype).itemsize
|
|
if self.scale_dtype is None:
|
|
return elem_bytes
|
|
return elem_bytes + segments * self.scale_dim(head_dim) * self.scale_store_dtype.itemsize
|
|
|
|
|
|
NONE = KVFormat("none", elem_dtype=None, elem_store_dtype=None)
|
|
FP8_PER_TOKEN_HEAD = KVFormat(
|
|
"fp8_per_token_head",
|
|
elem_dtype=torch.float8_e4m3fn,
|
|
elem_store_dtype=torch.uint8,
|
|
scale_dtype=torch.float32,
|
|
scale_store_dtype=torch.float32,
|
|
)
|
|
INT8_PER_TOKEN_HEAD = KVFormat(
|
|
"int8_per_token_head",
|
|
elem_dtype=torch.int8,
|
|
elem_store_dtype=torch.int8,
|
|
scale_dtype=torch.float32,
|
|
scale_store_dtype=torch.float32,
|
|
)
|
|
|
|
KV_FORMATS: dict[str, KVFormat] = {fmt.name: fmt for fmt in (NONE, FP8_PER_TOKEN_HEAD, INT8_PER_TOKEN_HEAD)}
|
|
# what a pool with per-row scale tensors and a reader built on kv_rows.py can declare
|
|
ROW_SCALE_FORMATS = frozenset({NONE.name, FP8_PER_TOKEN_HEAD.name, INT8_PER_TOKEN_HEAD.name})
|
|
|
|
|
|
def resolve_kv_format(kv_cache_dtype: str) -> KVFormat:
|
|
"""The format of a --kv-cache-dtype value; "auto" never quantizes."""
|
|
if kv_cache_dtype == "auto":
|
|
return NONE
|
|
try:
|
|
return KV_FORMATS[kv_cache_dtype]
|
|
except KeyError:
|
|
raise ValueError(f"unknown --kv-cache-dtype {kv_cache_dtype!r}; one of auto, {', '.join(KV_FORMATS)}") from None
|