mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(gguf): serve GLM-5.3-Flash GGUF checkpoints on glm5-next Adds the llama.cpp glm5-next GGUF reader (unsloth GLM-5.3-Flash-GGUF, both the glm5-next and the older glm5next arch spellings) on the GGUF quant kind from #690: - parse_gguf_config derives the text config from the GGUF KV metadata, with per-block attention.head_count_kv laying out the KDA/DSA hybrid, the trailing MTP block dropped like the HF loader drops it, and the MLA head widths from the *_length_mla keys. - iter_gguf_weights maps every tensor to the glm5_next state dict and undoes the converter transforms: A_log stored as -exp(A_log) is inverted, the per-head attn_k_b/attn_v_b pair (k transposed) fuses into the dense kv_b_proj layout the MLA absorption reads, and the q|k|v conv kernels concatenate on the channel axis. The mixed-type fused KDA in_proj (Q8_0 q|k|v + BF16 b|f_a|g_a) rides #702 segments; the routed experts load as GGUF expert banks at their shipped types (unsloth dynamic quants: IQ3_S gate/up + IQ4_XS down, Q8_0 outlier layers pass through natively). - passes quant_config to embed_tokens so the Q8_0 token_embd serves packed; registers Glm5NextGGUFForCausalLM and claims it in the AOT table. Verified end to end against unsloth GLM-5.3-Flash-UD-IQ4_XS (5 split shards): the full engine on a 32GB GPU serves coherent output with the experts off host banks. Text-only, TP=1. Part of #671. * fix(gguf): refuse unread scale tensors in glm5-next ggufs check_scales at config parse like the llama, qwen3 and gemma4 loaders (review on #721): a .scale / .input_scale outside an NVFP4 weight now fails with a clear error instead of loading silently unread. * fix(gguf): serve the quantized tensors of unsloth dynamic glm5-next quants unsloth's UD quants quantize tensors the AesSedAI-style files keep float: the MLA kv_b halves (Q8_0), the mHC mixers and the indexer's wk / weights_proj / compressor gate. kv_b_proj joins the registry's unquantized_modules (glm_moe_dsa keeps its kv_b dense for the MLA absorption for the same reason) and the reader dequantizes through the vendored gguf-py reference on every float-expecting read; wq_b keeps serving its checkpoint quant packed (attention.py keeps wk and weights_proj dense). Regression test stores the kv_b halves Q8_0. * fix(gguf): treat the glm5 pre-tokenizer spelling like glm4 llama-vocab.cpp maps both spellings to CHATGLM4 with ignore_merges; unsloth writes glm4 and AesSedAI's writer glm5 for the same vocabulary. Without the mapping a glm5 file loads but silently mis-tokenizes (tok_compare: 22,840/153,725 vocab words differ from the HF reference; 0 everywhere with it). * fix(gguf): size the mmq tail padding by ggml type and k pad_rows added one zero row of k columns, which covers the bytes MMQ reads past the last row only when k is at least half a padding unit (512); glm5-next's KDA f_b / g_b (k = linear head dim 128, Q8_0 in dynamic quants) are the first packed tensors below that and died at the first mul_mat. pad_rows now takes the ggml type and returns enough whole rows to cover mmq_tail_bytes for any (type, k), consistently in the reader's fused(), the gguf linear and embedding params and the expert host banks. Found serving the real unsloth UD-IQ4_XS: the dry load never runs a forward. * fix(gguf): take the older glm5-next writer variants in stride Two long-tail writer variants the full check sweep found (the GSQ-RCO lineage alone is ~57k downloads/month): - the conv kernels with a trailing unit dim: llama.cpp's glm5-next loader tries that 4-D shape before the 3-D one, so trailing unit dims in ne are dropped on float reads (they carry no elements). - attention.head_count_kv as the generic scalar (all blocks): wrong for the hybrid -- llama.cpp broadcasts it and then fails tensor creation -- so only the list is trusted and a scalar falls to the tensor-name census like the pre-list writer. * fix(gguf): glm5-next review cleanups - refuse instead of guessing metadata the model cannot express: the scalar attention.head_count_kv spelling (llama.cpp broadcasts it and then fails tensor creation; every public file carries the list), missing *_length_mla keys through need(), non-uniform swiglu_clamp_exp / swiglu_clamp_shexp, indexer.types with non-full layers, expert_gating_func other than sigmoid, and rope dims as a NotImplementedError instead of an assert. - call check_scales directly; aot_models comment on its own line. - drop the GGUF test files; coverage will be handled across families.
63 lines
2.6 KiB
Python
63 lines
2.6 KiB
Python
"""GGUF Linear: packed ggml rows multiplied by llama.cpp's MMVQ / MMQ kernels; a fused module runs one GEMM per ggml type, float parts as plain bf16 rows."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..ggml_types import is_quantized, pad_rows, row_bytes, segment_param, segments
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import QuantKind, gguf_types
|
|
from .base import LinearConfig, LinearKernel, LinearMethod
|
|
|
|
|
|
|
|
class GgmlLinearKernel(LinearKernel):
|
|
name = "ggml"
|
|
|
|
def unusable_reason(self, cfg: LinearConfig) -> str | None:
|
|
return None
|
|
|
|
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel import ggml
|
|
|
|
x2 = x.reshape(-1, x.shape[-1])
|
|
segs = layer._ggml_segments
|
|
m = x2.shape[0]
|
|
y = torch.empty(m, sum(s.rows for s in segs), dtype=x.dtype, device=x.device)
|
|
# every segment reads the same x: quantize it once for all the vector-kernel segments
|
|
xq = ggml.quantize_q8_1(x2) if sum(is_quantized(s.ggml_type) and ggml.uses_mmvq(s.ggml_type, m) for s in segs) > 1 else None
|
|
for i, s in enumerate(segs):
|
|
w = getattr(layer, segment_param(i))[: s.rows]
|
|
out = y[:, s.out_start : s.out_start + s.rows]
|
|
if is_quantized(s.ggml_type):
|
|
ggml.mul_mat(w, s.ggml_type, x2, out=out, xq=xq)
|
|
else:
|
|
torch.mm(x2, w.t(), out=out)
|
|
# llama.cpp multiplies an NVFP4 result by the tensor's weight_scale_2 (build_lora_mm)
|
|
if getattr(layer, "ggml_scale", None) is not None:
|
|
y.mul_(layer.ggml_scale)
|
|
if layer.bias is not None:
|
|
y = y + layer.bias
|
|
return y.reshape(*x.shape[:-1], y.shape[-1])
|
|
|
|
|
|
@register_method(QuantKind.GGUF, LayerKind.LINEAR)
|
|
class GGUFLinearMethod(LinearMethod):
|
|
candidates = (GgmlLinearKernel,)
|
|
|
|
def create_weights(self, layer: Any) -> None:
|
|
g = self.cfg
|
|
segs = segments(gguf_types(g.scheme), g.output_sizes)
|
|
layer._ggml_segments = segs
|
|
k = g.in_features
|
|
for i, s in enumerate(segs):
|
|
if is_quantized(s.ggml_type):
|
|
setattr(layer, segment_param(i), torch.empty(s.rows + pad_rows(s.ggml_type, k), row_bytes(s.ggml_type, k), dtype=torch.uint8))
|
|
else:
|
|
setattr(layer, segment_param(i), torch.empty(s.rows, k))
|
|
if g.scheme.weight.scale is not None:
|
|
# one value per output row: each part's weight_scale_2, 1 for parts without one
|
|
layer.ggml_scale = torch.empty(sum(s.rows for s in segs), dtype=torch.float32)
|