mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
34 lines
1.2 KiB
Python
34 lines
1.2 KiB
Python
"""Dense tables: the indexing kernel looks rows up; a tied head runs the unquantized linear kernel on the same table."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..linear.unquantized import TorchLinearKernel
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import QuantKind
|
|
from .base import EmbeddingKernel, EmbeddingMethod
|
|
|
|
|
|
class IndexingEmbeddingKernel(EmbeddingKernel):
|
|
name = "indexing"
|
|
|
|
def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel import indexing
|
|
|
|
return indexing(weights=layer.weight, indices=ids, vocab_range=layer.vocab_range if layer.tp_size > 1 else None)
|
|
|
|
def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
|
return TorchLinearKernel().apply(SimpleNamespace(weight=layer.weight, bias=bias), x)
|
|
|
|
|
|
@register_method(QuantKind.NONE, LayerKind.EMBEDDING)
|
|
class UnquantizedEmbeddingMethod(EmbeddingMethod):
|
|
candidates = (IndexingEmbeddingKernel,)
|
|
|
|
def create_weights(self, layer: Any) -> None:
|
|
layer.weight = torch.empty(self.cfg.num_embeddings, self.cfg.embedding_dim)
|