mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
61 lines
1.6 KiB
Python
61 lines
1.6 KiB
Python
"""Two registries: checkpoint dialects, and (kind, layer kind) -> Method."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from enum import Enum
|
|
from typing import Any
|
|
|
|
from .scheme import QuantKind
|
|
|
|
|
|
class LayerKind(Enum):
|
|
LINEAR = "linear"
|
|
MOE = "moe"
|
|
EMBEDDING = "embedding"
|
|
|
|
def __str__(self) -> str:
|
|
return self.value
|
|
|
|
|
|
_DIALECTS: dict[str, type] = {}
|
|
_METHODS: dict[tuple[QuantKind, LayerKind], type] = {}
|
|
|
|
|
|
def register_dialect(cls: type) -> type:
|
|
name = cls.dialect
|
|
if name in _DIALECTS:
|
|
raise ValueError(f"quant dialect {name!r} already registered: {_DIALECTS[name].__name__}")
|
|
_DIALECTS[name] = cls
|
|
return cls
|
|
|
|
|
|
def dialects() -> tuple[type, ...]:
|
|
return tuple(_DIALECTS.values())
|
|
|
|
|
|
def register_method(kind: QuantKind, layer_kind: LayerKind):
|
|
if not isinstance(kind, QuantKind) or not isinstance(layer_kind, LayerKind):
|
|
raise TypeError(f"cannot register quant method for {layer_kind!r}.{kind!r}")
|
|
|
|
def deco(cls: type) -> type:
|
|
key = (kind, layer_kind)
|
|
if key in _METHODS:
|
|
raise ValueError(f"quant method for {layer_kind}.{kind} already registered: {_METHODS[key].__name__}")
|
|
cls.kind = kind
|
|
cls.layer_kind = layer_kind
|
|
_METHODS[key] = cls
|
|
return cls
|
|
|
|
return deco
|
|
|
|
|
|
def method_class(kind: QuantKind, layer_kind: LayerKind) -> type[Any]:
|
|
try:
|
|
return _METHODS[(kind, layer_kind)]
|
|
except KeyError:
|
|
raise NotImplementedError(f"no quant method for {layer_kind}.{kind}") from None
|
|
|
|
|
|
def methods_for(layer_kind: LayerKind) -> dict[QuantKind, type]:
|
|
return {kind: cls for (kind, lk), cls in _METHODS.items() if lk == layer_kind}
|