Files
Xiaoze Fanandzihaomu e5d3bd3faf feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding

* build(deps): vendor the gguf-py reader

* feat(quant): add the GGUF quant kind and an embedding quant method

* feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path

* fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA

* perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16

* build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small

* fix(gguf): read F32, F16 and BF16 PLE tables without ggml

* fix(moe): pad the offload slot cache for ggml MMQ tail reads

* fix(gguf): support hip builds and runtime devices

* fix(gguf): copy same-type dequantize and check for missing ggml converters

* fix(gguf): load quantized qwen4exp ple_key and ple_value

* feat(gguf): run float parts of a fused GGUF linear as bf16 segments

* fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights

* fix(moe): zero the prefill staging area after the last prefill layer

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-15 16:33:16 -07:00

65 lines
2.0 KiB
Python

"""The embedding side: per-layer config, the kernel ABC and the Method base. The method owns the vocab table: it looks rows up, and a tied LM head multiplies by the same table through it."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
import torch
from ..method import QuantMethod
from ..scheme import QuantScheme
@dataclass(frozen=True)
class EmbeddingConfig:
"""The TP-local table shape, the dtype a lookup returns, and the table's scheme."""
num_embeddings: int
embedding_dim: int
tp_size: int
dtype: torch.dtype
scheme: QuantScheme | None = None
@classmethod
def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "EmbeddingConfig":
return cls(layer.num_embeddings_tp, layer.embedding_dim, layer.tp_size, torch.get_default_dtype(), scheme)
class EmbeddingKernel(ABC):
"""One backend for one table kind: the row lookup, and ``x @ table^T`` for a tied LM head."""
name: ClassVar[str]
def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:
return None
def worth_it(self, cfg: EmbeddingConfig) -> bool:
return True
def finalize(self, layer: Any) -> None:
pass
@abstractmethod
def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor: ...
@abstractmethod
def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: ...
class EmbeddingMethod(QuantMethod):
"""Declares the table for its kind; the lookup and the tied head's product go to the kernel."""
@abstractmethod
def create_weights(self, layer: Any) -> None: ...
def finalize(self, layer: Any) -> None:
self.kernel.finalize(layer)
def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
return self.kernel.embed(layer, ids)
def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
return self.kernel.logits(layer, x, bias)