Files
FreeToken/python/freetoken/layers/quantization/registry.py
Xiaoze Fanandzihaomu e5d3bd3faf feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding

* build(deps): vendor the gguf-py reader

* feat(quant): add the GGUF quant kind and an embedding quant method

* feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path

* fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA

* perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16

* build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small

* fix(gguf): read F32, F16 and BF16 PLE tables without ggml

* fix(moe): pad the offload slot cache for ggml MMQ tail reads

* fix(gguf): support hip builds and runtime devices

* fix(gguf): copy same-type dequantize and check for missing ggml converters

* fix(gguf): load quantized qwen4exp ple_key and ple_value

* feat(gguf): run float parts of a fused GGUF linear as bf16 segments

* fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights

* fix(moe): zero the prefill staging area after the last prefill layer

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-15 16:33:16 -07:00

61 lines
1.6 KiB
Python

"""Two registries: checkpoint dialects, and (kind, layer kind) -> Method."""
from __future__ import annotations
from enum import Enum
from typing import Any
from .scheme import QuantKind
class LayerKind(Enum):
LINEAR = "linear"
MOE = "moe"
EMBEDDING = "embedding"
def __str__(self) -> str:
return self.value
_DIALECTS: dict[str, type] = {}
_METHODS: dict[tuple[QuantKind, LayerKind], type] = {}
def register_dialect(cls: type) -> type:
name = cls.dialect
if name in _DIALECTS:
raise ValueError(f"quant dialect {name!r} already registered: {_DIALECTS[name].__name__}")
_DIALECTS[name] = cls
return cls
def dialects() -> tuple[type, ...]:
return tuple(_DIALECTS.values())
def register_method(kind: QuantKind, layer_kind: LayerKind):
if not isinstance(kind, QuantKind) or not isinstance(layer_kind, LayerKind):
raise TypeError(f"cannot register quant method for {layer_kind!r}.{kind!r}")
def deco(cls: type) -> type:
key = (kind, layer_kind)
if key in _METHODS:
raise ValueError(f"quant method for {layer_kind}.{kind} already registered: {_METHODS[key].__name__}")
cls.kind = kind
cls.layer_kind = layer_kind
_METHODS[key] = cls
return cls
return deco
def method_class(kind: QuantKind, layer_kind: LayerKind) -> type[Any]:
try:
return _METHODS[(kind, layer_kind)]
except KeyError:
raise NotImplementedError(f"no quant method for {layer_kind}.{kind}") from None
def methods_for(layer_kind: LayerKind) -> dict[QuantKind, type]:
return {kind: cls for (kind, lk), cls in _METHODS.items() if lk == layer_kind}