mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
98 lines
3.3 KiB
Python
98 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
import torch
|
|
|
|
|
|
def _use_flashinfer(x: torch.Tensor, out: torch.Tensor | None) -> bool:
|
|
"""flashinfer's *_and_mul kernels take fp16 / bf16 in and the same dtype out; the Triton kernels take any float dtypes."""
|
|
import torch
|
|
|
|
from freetoken.kernel.backend import is_flashinfer_installed
|
|
|
|
return x.dtype in (torch.float16, torch.bfloat16) and (out is None or out.dtype == x.dtype) and is_flashinfer_installed()
|
|
|
|
|
|
def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
|
|
if _use_flashinfer(x, out):
|
|
from flashinfer import silu_and_mul
|
|
else:
|
|
from freetoken.kernel.triton.activation import silu_and_mul
|
|
|
|
return silu_and_mul(x, out=out)
|
|
|
|
|
|
def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
|
|
if _use_flashinfer(x, out):
|
|
from flashinfer import gelu_and_mul
|
|
else:
|
|
from freetoken.kernel.triton.activation import gelu_and_mul
|
|
|
|
return gelu_and_mul(x, out=out)
|
|
|
|
|
|
def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
|
|
"""tanh-approximate GELU gate (`gelu_pytorch_tanh`) followed by elementwise mul."""
|
|
if _use_flashinfer(x, out):
|
|
from flashinfer import gelu_tanh_and_mul
|
|
else:
|
|
from freetoken.kernel.triton.activation import gelu_tanh_and_mul
|
|
|
|
return gelu_tanh_and_mul(x, out=out)
|
|
|
|
|
|
def swigluoai_and_mul(
|
|
x: torch.Tensor,
|
|
out: torch.Tensor | None = None,
|
|
*,
|
|
alpha: float = 1.702,
|
|
limit: float = 7.0,
|
|
):
|
|
"""SwiGLU-OAI (gpt-oss / MiniMax-M3 ``swigluoai``) over UNINTERLEAVED halves
|
|
(gate ``x[..., :d]``, up ``x[..., d:]``): ``clamp(gate, max=limit) *
|
|
sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``. Always the in-repo Triton
|
|
kernel (flashinfer ships no clamped-swiglu *_and_mul)."""
|
|
from freetoken.kernel.triton.activation import swigluoai_and_mul
|
|
|
|
return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)
|
|
|
|
|
|
def swiglu_clamp_and_mul(
|
|
x, out=None, *, alpha: float = 1.0, limit: float = 10.0
|
|
):
|
|
"""GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""
|
|
from freetoken.kernel.triton.activation import swiglu_clamp_and_mul
|
|
|
|
return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)
|
|
|
|
|
|
GATED_ACTIVATIONS = ("silu", "gelu", "gelu_tanh", "swigluoai", "swiglu_clamp")
|
|
|
|
|
|
def gated_act_and_mul(activation: str, x, out, *, alpha: float = 1.0, limit: float = float("inf")):
|
|
"""The epilogue between the two expert GEMMs: ``activation`` over uninterleaved [gate; up] halves into ``out``."""
|
|
if activation == "silu":
|
|
return silu_and_mul(x, out)
|
|
if activation == "gelu":
|
|
return gelu_and_mul(x, out)
|
|
if activation == "gelu_tanh":
|
|
return gelu_tanh_and_mul(x, out)
|
|
if activation == "swigluoai":
|
|
return swigluoai_and_mul(x, out, alpha=alpha, limit=limit)
|
|
if activation == "swiglu_clamp":
|
|
return swiglu_clamp_and_mul(x, out, alpha=alpha, limit=limit)
|
|
raise ValueError(f"no gated activation {activation!r}; known: {GATED_ACTIVATIONS}")
|
|
|
|
|
|
__all__ = [
|
|
"silu_and_mul",
|
|
"gelu_and_mul",
|
|
"gelu_tanh_and_mul",
|
|
"swigluoai_and_mul",
|
|
"swiglu_clamp_and_mul",
|
|
"gated_act_and_mul",
|
|
"GATED_ACTIVATIONS",
|
|
]
|