Files

121 lines
4.4 KiB
Python

"""QuantScheme: the storage description and role set of one quantized weight, plus the per-kind builders."""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from typing import Iterable
import torch
# -1 selects the whole dimension: (-1, -1) one scale per tensor, (1, -1) one per output row, (1, 16) one per 16 input elements, (128, 128) a 2-D block
GroupShape = tuple[int, int]
class QuantKind(Enum):
"""The quantization kinds; one Method per (kind, layer kind)."""
NONE = "none"
FP8_TENSOR = "fp8_tensor"
FP8_BLOCK = "fp8_block"
MXFP8 = "mxfp8"
NVFP4 = "nvfp4"
MXFP4 = "mxfp4"
GGUF = "gguf"
def __str__(self) -> str:
return self.value
FP8_BLOCK = 128 # DeepSeek-V3 style square block; V4.1 exports 32x32
FP8_BLOCK_SIZES = (32, 128)
NVFP4_GROUP = 16
MX_GROUP = 32
@dataclass(frozen=True)
class WeightDesc:
elem: str
group: GroupShape | None
scale: str | None
@dataclass(frozen=True, eq=False)
class QuantScheme:
"""``roles`` are the canonical tensor names the kind's Method declares; which checkpoint tensors feed them is the dialect Config's business."""
kind: QuantKind
weight: WeightDesc
roles: frozenset[str]
def __init__(self, kind: QuantKind, weight: WeightDesc, roles: Iterable[str]):
if not isinstance(kind, QuantKind):
raise TypeError(f"kind must be a QuantKind, got {kind!r}")
object.__setattr__(self, "kind", kind)
object.__setattr__(self, "weight", weight)
object.__setattr__(self, "roles", frozenset(roles))
def _key(self):
return (self.kind, self.weight, self.roles)
def __eq__(self, other: object) -> bool:
return isinstance(other, QuantScheme) and self._key() == other._key()
def __hash__(self) -> int:
return hash(self._key())
def has(self, role: str) -> bool:
return role in self.roles
def __repr__(self) -> str:
return f"QuantScheme({self.kind}, {self.weight}, roles={sorted(self.roles)})"
# ---------------------------------------------------------------------------
# Builders: one per kind; the dialect configs pick the variant their checkpoints carry
# ---------------------------------------------------------------------------
def fp8_tensor_scheme(scale: str, *, per_row: bool = False, input_scale: bool = False) -> QuantScheme:
roles = {"weight", "weight_scale"} | ({"input_scale"} if input_scale else set())
return QuantScheme(QuantKind.FP8_TENSOR, WeightDesc("e4m3", (1, -1) if per_row else (-1, -1), scale), roles)
def fp8_block_scheme(scale: str, block: int = FP8_BLOCK) -> QuantScheme:
"""Square ``block x block`` e4m3 weight blocks, one scale each; ``block`` is the checkpoint's ``weight_block_size``."""
if block not in FP8_BLOCK_SIZES:
raise NotImplementedError(f"fp8 block size {block} is not supported; one of {FP8_BLOCK_SIZES}")
return QuantScheme(QuantKind.FP8_BLOCK, WeightDesc("e4m3", (block, block), scale), {"weight", "weight_scale_inv"})
def fp8_block_size(scheme: QuantScheme) -> int:
"""The square block edge of an FP8_BLOCK scheme (128 for DeepSeek-V3 style exports, 32 for DeepSeek-V4.1)."""
assert scheme.kind is QuantKind.FP8_BLOCK, scheme
return scheme.weight.group[0]
def mxfp8_scheme() -> QuantScheme:
return QuantScheme(QuantKind.MXFP8, WeightDesc("e4m3", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale_inv"})
def nvfp4_scheme(*, input_scale: bool) -> QuantScheme:
roles = {"weight", "weight_scale", "weight_global"} | ({"input_scale"} if input_scale else set())
return QuantScheme(QuantKind.NVFP4, WeightDesc("e2m1", (1, NVFP4_GROUP), "e4m3"), roles)
def mxfp4_scheme() -> QuantScheme:
return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})
def gguf_scheme(types: Iterable[int], scaled: bool = False) -> QuantScheme:
"""Packed ggml blocks, one ggml type per fused part (``types`` follows the parts' order).
``scaled``: the NVFP4 tensors carry the fp32 ``<name>.scale`` (weight_scale_2) that llama.cpp's converter writes."""
return QuantScheme(QuantKind.GGUF, WeightDesc("ggml:" + ",".join(str(int(t)) for t in types), None, "f32" if scaled else None), {"weight"})
def gguf_types(scheme: QuantScheme) -> tuple[int, ...]:
assert scheme.kind is QuantKind.GGUF, scheme
return tuple(int(t) for t in scheme.weight.elem.removeprefix("ggml:").split(","))