mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(gguf): read NVFP4 token_embd rows in the vendored get_rows Signed-off-by: iamanishx <manishbiswal754@gmail.com> * fix(tests) Signed-off-by: iamanishx <manishbiswal754@gmail.com> --------- Signed-off-by: iamanishx <manishbiswal754@gmail.com>
49 lines
2.0 KiB
Python
49 lines
2.0 KiB
Python
"""GGUF tables: packed ggml rows, decoded per looked-up row; a tied head runs the GGUF linear kernel on the same rows."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..ggml_types import Segment, pad_rows, row_bytes
|
|
from ..linear.gguf import GgmlLinearKernel
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import QuantKind, gguf_types
|
|
from .base import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod
|
|
|
|
|
|
class GgmlEmbeddingKernel(EmbeddingKernel):
|
|
name = "ggml"
|
|
|
|
def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:
|
|
if cfg.tp_size > 1:
|
|
return "GGUF tables are not sharded over TP"
|
|
# llama.cpp reads no token_embd scale and refuses a tied NVFP4 head that has one
|
|
return "GGUF tables with a weight scale are not supported" if cfg.scheme.weight.scale is not None else None
|
|
|
|
def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel import ggml
|
|
|
|
cfg = layer.quant_method.cfg
|
|
(ggml_type,) = gguf_types(cfg.scheme)
|
|
rows = ggml.get_rows(layer.weight, ggml_type, cfg.embedding_dim, ids.reshape(-1).to(torch.int32), cfg.dtype)
|
|
return rows.view(*ids.shape, cfg.embedding_dim)
|
|
|
|
def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
|
cfg = layer.quant_method.cfg
|
|
(ggml_type,) = gguf_types(cfg.scheme)
|
|
table = SimpleNamespace(_ggml_segments=[Segment(ggml_type, 0, 1, 0, cfg.num_embeddings)], weight=layer.weight, bias=bias)
|
|
return GgmlLinearKernel().apply(table, x)
|
|
|
|
|
|
@register_method(QuantKind.GGUF, LayerKind.EMBEDDING)
|
|
class GGUFEmbeddingMethod(EmbeddingMethod):
|
|
candidates = (GgmlEmbeddingKernel,)
|
|
|
|
def create_weights(self, layer: Any) -> None:
|
|
(ggml_type,) = gguf_types(self.cfg.scheme)
|
|
h = self.cfg.embedding_dim
|
|
layer.weight = torch.empty(self.cfg.num_embeddings + pad_rows(ggml_type, h), row_bytes(ggml_type, h), dtype=torch.uint8)
|