Files
Xiaoze Fan 2fb371e318 feat(kvcache): add fp8_ds_mla and nvfp4_ds_mla kv formats for deepseek-v4 / v4.1 (#920)
* feat(kvcache): add fp8_ds_mla and nvfp4_ds_mla kv formats for deepseek-v4 / v4.1

Window and compressed KV are stored as fp8_ds_mla rows (584 B instead of
1024 B) and indexer keys as MXFP4, holding the same values as the
reference's quant round-trip. DeepSeek-V4.1's packed layout is named
nvfp4_ds_mla and shares the row-format code.

* refactor(engine): drop the model field from the dsv4 kv formats
2026-11-18 22:04:03 -08:00

98 lines
4.2 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
import torch
ROW = -1 # one scale per row
@dataclass(frozen=True)
class KVFormat:
"""How one K or V row is stored in the KV pool; the encoding lives in the ``kv_rows.py`` arms of the same name."""
name: str # --kv-cache-dtype value and the kernels' KV_FMT
elem_dtype: torch.dtype | None # logical element type; None follows the compute dtype
elem_store_dtype: torch.dtype | None # dtype of the element tensor; None follows the compute dtype
scale_dtype: torch.dtype | None = None # logical scale type; None means no scale tensor
scale_store_dtype: torch.dtype | None = None # dtype of the scale tensor
scale_group: int = ROW # channels per scale
def __post_init__(self) -> None:
if (self.elem_dtype is None) != (self.elem_store_dtype is None):
raise ValueError(f"{self.name}: elem_dtype and elem_store_dtype must be set together")
if (self.scale_dtype is None) != (self.scale_store_dtype is None):
raise ValueError(f"{self.name}: scale_dtype and scale_store_dtype must be set together")
if self.scale_dtype is not None and self.scale_dtype.itemsize != self.scale_store_dtype.itemsize:
raise ValueError(f"{self.name}: scale_store_dtype must have the size of scale_dtype")
@property
def quantized(self) -> bool:
return self.elem_dtype is not None
def elem_tensor_dtype(self, compute_dtype: torch.dtype) -> torch.dtype:
return compute_dtype if self.elem_store_dtype is None else self.elem_store_dtype
def elem_dim(self, head_dim: int) -> int:
"""Elements of the element tensor per row."""
if self.elem_dtype is None:
return head_dim
row_bytes = head_dim * self.elem_dtype.itemsize
if row_bytes % self.elem_store_dtype.itemsize:
raise ValueError(f"{self.name} cannot store a {head_dim}-wide row")
return row_bytes // self.elem_store_dtype.itemsize
def scale_dim(self, head_dim: int) -> int:
"""Scales per row."""
if self.scale_dtype is None:
return 0
if self.scale_group == ROW:
return 1
if head_dim % self.scale_group:
raise ValueError(f"{self.name} needs head_dim divisible by {self.scale_group}, got {head_dim}")
return head_dim // self.scale_group
def row_bytes(self, head_dim: int, compute_dtype: torch.dtype, segments: int = 1) -> int:
"""Bytes of one K or V row split into ``segments`` separately scaled parts, its scales included."""
elem_bytes = self.elem_dim(head_dim) * self.elem_tensor_dtype(compute_dtype).itemsize
if self.scale_dtype is None:
return elem_bytes
return elem_bytes + segments * self.scale_dim(head_dim) * self.scale_store_dtype.itemsize
NONE = KVFormat("none", elem_dtype=None, elem_store_dtype=None)
FP8_PER_TOKEN_HEAD = KVFormat(
"fp8_per_token_head",
elem_dtype=torch.float8_e4m3fn,
elem_store_dtype=torch.uint8,
scale_dtype=torch.float32,
scale_store_dtype=torch.float32,
)
INT8_PER_TOKEN_HEAD = KVFormat(
"int8_per_token_head",
elem_dtype=torch.int8,
elem_store_dtype=torch.int8,
scale_dtype=torch.float32,
scale_store_dtype=torch.float32,
)
# DeepSeek-V4 / V4.1 layouts, encoded by their own pools (per-tier formats in kvcache/dsv4/row_format.py); names follow vLLM
FP8_DS_MLA = KVFormat("fp8_ds_mla", elem_dtype=None, elem_store_dtype=None)
NVFP4_DS_MLA = KVFormat("nvfp4_ds_mla", elem_dtype=None, elem_store_dtype=None)
KV_FORMATS: dict[str, KVFormat] = {
fmt.name: fmt for fmt in (NONE, FP8_PER_TOKEN_HEAD, INT8_PER_TOKEN_HEAD, FP8_DS_MLA, NVFP4_DS_MLA)
}
# what a pool with per-row scale tensors and a reader built on kv_rows.py can declare
ROW_SCALE_FORMATS = frozenset({NONE.name, FP8_PER_TOKEN_HEAD.name, INT8_PER_TOKEN_HEAD.name})
def resolve_kv_format(kv_cache_dtype: str) -> KVFormat:
"""The format of a --kv-cache-dtype value; "auto" never quantizes."""
if kv_cache_dtype == "auto":
return NONE
try:
return KV_FORMATS[kv_cache_dtype]
except KeyError:
raise ValueError(f"unknown --kv-cache-dtype {kv_cache_dtype!r}; one of auto, {', '.join(KV_FORMATS)}") from None