mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
65 lines
2.0 KiB
Python
65 lines
2.0 KiB
Python
"""The embedding side: per-layer config, the kernel ABC and the Method base. The method owns the vocab table: it looks rows up, and a tied LM head multiplies by the same table through it."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from typing import Any, ClassVar
|
|
|
|
import torch
|
|
|
|
from ..method import QuantMethod
|
|
from ..scheme import QuantScheme
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EmbeddingConfig:
|
|
"""The TP-local table shape, the dtype a lookup returns, and the table's scheme."""
|
|
|
|
num_embeddings: int
|
|
embedding_dim: int
|
|
tp_size: int
|
|
dtype: torch.dtype
|
|
scheme: QuantScheme | None = None
|
|
|
|
@classmethod
|
|
def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "EmbeddingConfig":
|
|
return cls(layer.num_embeddings_tp, layer.embedding_dim, layer.tp_size, torch.get_default_dtype(), scheme)
|
|
|
|
|
|
class EmbeddingKernel(ABC):
|
|
"""One backend for one table kind: the row lookup, and ``x @ table^T`` for a tied LM head."""
|
|
|
|
name: ClassVar[str]
|
|
|
|
def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:
|
|
return None
|
|
|
|
def worth_it(self, cfg: EmbeddingConfig) -> bool:
|
|
return True
|
|
|
|
def finalize(self, layer: Any) -> None:
|
|
pass
|
|
|
|
@abstractmethod
|
|
def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor: ...
|
|
|
|
@abstractmethod
|
|
def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: ...
|
|
|
|
|
|
class EmbeddingMethod(QuantMethod):
|
|
"""Declares the table for its kind; the lookup and the tied head's product go to the kernel."""
|
|
|
|
@abstractmethod
|
|
def create_weights(self, layer: Any) -> None: ...
|
|
|
|
def finalize(self, layer: Any) -> None:
|
|
self.kernel.finalize(layer)
|
|
|
|
def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
|
|
return self.kernel.embed(layer, ids)
|
|
|
|
def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
|
return self.kernel.logits(layer, x, bias)
|