Files
Xiaoze Fanandzihaomu e5d3bd3faf feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding

* build(deps): vendor the gguf-py reader

* feat(quant): add the GGUF quant kind and an embedding quant method

* feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path

* fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA

* perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16

* build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small

* fix(gguf): read F32, F16 and BF16 PLE tables without ggml

* fix(moe): pad the offload slot cache for ggml MMQ tail reads

* fix(gguf): support hip builds and runtime devices

* fix(gguf): copy same-type dequantize and check for missing ggml converters

* fix(gguf): load quantized qwen4exp ple_key and ple_value

* feat(gguf): run float parts of a fused GGUF linear as bf16 segments

* fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights

* fix(moe): zero the prefill staging area after the last prefill layer

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-15 16:33:16 -07:00

98 lines
3.3 KiB
Python

from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import torch
def _use_flashinfer(x: torch.Tensor, out: torch.Tensor | None) -> bool:
"""flashinfer's *_and_mul kernels take fp16 / bf16 in and the same dtype out; the Triton kernels take any float dtypes."""
import torch
from freetoken.kernel.backend import is_flashinfer_installed
return x.dtype in (torch.float16, torch.bfloat16) and (out is None or out.dtype == x.dtype) and is_flashinfer_installed()
def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
if _use_flashinfer(x, out):
from flashinfer import silu_and_mul
else:
from freetoken.kernel.triton.activation import silu_and_mul
return silu_and_mul(x, out=out)
def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
if _use_flashinfer(x, out):
from flashinfer import gelu_and_mul
else:
from freetoken.kernel.triton.activation import gelu_and_mul
return gelu_and_mul(x, out=out)
def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
"""tanh-approximate GELU gate (`gelu_pytorch_tanh`) followed by elementwise mul."""
if _use_flashinfer(x, out):
from flashinfer import gelu_tanh_and_mul
else:
from freetoken.kernel.triton.activation import gelu_tanh_and_mul
return gelu_tanh_and_mul(x, out=out)
def swigluoai_and_mul(
x: torch.Tensor,
out: torch.Tensor | None = None,
*,
alpha: float = 1.702,
limit: float = 7.0,
):
"""SwiGLU-OAI (gpt-oss / MiniMax-M3 ``swigluoai``) over UNINTERLEAVED halves
(gate ``x[..., :d]``, up ``x[..., d:]``): ``clamp(gate, max=limit) *
sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``. Always the in-repo Triton
kernel (flashinfer ships no clamped-swiglu *_and_mul)."""
from freetoken.kernel.triton.activation import swigluoai_and_mul
return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)
def swiglu_clamp_and_mul(
x, out=None, *, alpha: float = 1.0, limit: float = 10.0
):
"""GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""
from freetoken.kernel.triton.activation import swiglu_clamp_and_mul
return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)
GATED_ACTIVATIONS = ("silu", "gelu", "gelu_tanh", "swigluoai", "swiglu_clamp")
def gated_act_and_mul(activation: str, x, out, *, alpha: float = 1.0, limit: float = float("inf")):
"""The epilogue between the two expert GEMMs: ``activation`` over uninterleaved [gate; up] halves into ``out``."""
if activation == "silu":
return silu_and_mul(x, out)
if activation == "gelu":
return gelu_and_mul(x, out)
if activation == "gelu_tanh":
return gelu_tanh_and_mul(x, out)
if activation == "swigluoai":
return swigluoai_and_mul(x, out, alpha=alpha, limit=limit)
if activation == "swiglu_clamp":
return swiglu_clamp_and_mul(x, out, alpha=alpha, limit=limit)
raise ValueError(f"no gated activation {activation!r}; known: {GATED_ACTIVATIONS}")
__all__ = [
"silu_and_mul",
"gelu_and_mul",
"gelu_tanh_and_mul",
"swigluoai_and_mul",
"swiglu_clamp_and_mul",
"gated_act_and_mul",
"GATED_ACTIVATIONS",
]