mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(kvcache): add fp8_ds_mla and nvfp4_ds_mla kv formats for deepseek-v4 / v4.1 Window and compressed KV are stored as fp8_ds_mla rows (584 B instead of 1024 B) and indexer keys as MXFP4, holding the same values as the reference's quant round-trip. DeepSeek-V4.1's packed layout is named nvfp4_ds_mla and shares the row-format code. * refactor(engine): drop the model field from the dsv4 kv formats
98 lines
4.2 KiB
Python
98 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import torch
|
|
|
|
ROW = -1 # one scale per row
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class KVFormat:
|
|
"""How one K or V row is stored in the KV pool; the encoding lives in the ``kv_rows.py`` arms of the same name."""
|
|
|
|
name: str # --kv-cache-dtype value and the kernels' KV_FMT
|
|
elem_dtype: torch.dtype | None # logical element type; None follows the compute dtype
|
|
elem_store_dtype: torch.dtype | None # dtype of the element tensor; None follows the compute dtype
|
|
scale_dtype: torch.dtype | None = None # logical scale type; None means no scale tensor
|
|
scale_store_dtype: torch.dtype | None = None # dtype of the scale tensor
|
|
scale_group: int = ROW # channels per scale
|
|
|
|
def __post_init__(self) -> None:
|
|
if (self.elem_dtype is None) != (self.elem_store_dtype is None):
|
|
raise ValueError(f"{self.name}: elem_dtype and elem_store_dtype must be set together")
|
|
if (self.scale_dtype is None) != (self.scale_store_dtype is None):
|
|
raise ValueError(f"{self.name}: scale_dtype and scale_store_dtype must be set together")
|
|
if self.scale_dtype is not None and self.scale_dtype.itemsize != self.scale_store_dtype.itemsize:
|
|
raise ValueError(f"{self.name}: scale_store_dtype must have the size of scale_dtype")
|
|
|
|
@property
|
|
def quantized(self) -> bool:
|
|
return self.elem_dtype is not None
|
|
|
|
def elem_tensor_dtype(self, compute_dtype: torch.dtype) -> torch.dtype:
|
|
return compute_dtype if self.elem_store_dtype is None else self.elem_store_dtype
|
|
|
|
def elem_dim(self, head_dim: int) -> int:
|
|
"""Elements of the element tensor per row."""
|
|
if self.elem_dtype is None:
|
|
return head_dim
|
|
row_bytes = head_dim * self.elem_dtype.itemsize
|
|
if row_bytes % self.elem_store_dtype.itemsize:
|
|
raise ValueError(f"{self.name} cannot store a {head_dim}-wide row")
|
|
return row_bytes // self.elem_store_dtype.itemsize
|
|
|
|
def scale_dim(self, head_dim: int) -> int:
|
|
"""Scales per row."""
|
|
if self.scale_dtype is None:
|
|
return 0
|
|
if self.scale_group == ROW:
|
|
return 1
|
|
if head_dim % self.scale_group:
|
|
raise ValueError(f"{self.name} needs head_dim divisible by {self.scale_group}, got {head_dim}")
|
|
return head_dim // self.scale_group
|
|
|
|
def row_bytes(self, head_dim: int, compute_dtype: torch.dtype, segments: int = 1) -> int:
|
|
"""Bytes of one K or V row split into ``segments`` separately scaled parts, its scales included."""
|
|
elem_bytes = self.elem_dim(head_dim) * self.elem_tensor_dtype(compute_dtype).itemsize
|
|
if self.scale_dtype is None:
|
|
return elem_bytes
|
|
return elem_bytes + segments * self.scale_dim(head_dim) * self.scale_store_dtype.itemsize
|
|
|
|
|
|
NONE = KVFormat("none", elem_dtype=None, elem_store_dtype=None)
|
|
FP8_PER_TOKEN_HEAD = KVFormat(
|
|
"fp8_per_token_head",
|
|
elem_dtype=torch.float8_e4m3fn,
|
|
elem_store_dtype=torch.uint8,
|
|
scale_dtype=torch.float32,
|
|
scale_store_dtype=torch.float32,
|
|
)
|
|
INT8_PER_TOKEN_HEAD = KVFormat(
|
|
"int8_per_token_head",
|
|
elem_dtype=torch.int8,
|
|
elem_store_dtype=torch.int8,
|
|
scale_dtype=torch.float32,
|
|
scale_store_dtype=torch.float32,
|
|
)
|
|
|
|
# DeepSeek-V4 / V4.1 layouts, encoded by their own pools (per-tier formats in kvcache/dsv4/row_format.py); names follow vLLM
|
|
FP8_DS_MLA = KVFormat("fp8_ds_mla", elem_dtype=None, elem_store_dtype=None)
|
|
NVFP4_DS_MLA = KVFormat("nvfp4_ds_mla", elem_dtype=None, elem_store_dtype=None)
|
|
|
|
KV_FORMATS: dict[str, KVFormat] = {
|
|
fmt.name: fmt for fmt in (NONE, FP8_PER_TOKEN_HEAD, INT8_PER_TOKEN_HEAD, FP8_DS_MLA, NVFP4_DS_MLA)
|
|
}
|
|
# what a pool with per-row scale tensors and a reader built on kv_rows.py can declare
|
|
ROW_SCALE_FORMATS = frozenset({NONE.name, FP8_PER_TOKEN_HEAD.name, INT8_PER_TOKEN_HEAD.name})
|
|
|
|
|
|
def resolve_kv_format(kv_cache_dtype: str) -> KVFormat:
|
|
"""The format of a --kv-cache-dtype value; "auto" never quantizes."""
|
|
if kv_cache_dtype == "auto":
|
|
return NONE
|
|
try:
|
|
return KV_FORMATS[kv_cache_dtype]
|
|
except KeyError:
|
|
raise ValueError(f"unknown --kv-cache-dtype {kv_cache_dtype!r}; one of auto, {', '.join(KV_FORMATS)}") from None
|