from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: import torch def _use_flashinfer(x: torch.Tensor, out: torch.Tensor | None) -> bool: """flashinfer's *_and_mul kernels take fp16 / bf16 in and the same dtype out; the Triton kernels take any float dtypes.""" import torch from freetoken.kernel.backend import is_flashinfer_installed return x.dtype in (torch.float16, torch.bfloat16) and (out is None or out.dtype == x.dtype) and is_flashinfer_installed() def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None): if _use_flashinfer(x, out): from flashinfer import silu_and_mul else: from freetoken.kernel.triton.activation import silu_and_mul return silu_and_mul(x, out=out) def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None): if _use_flashinfer(x, out): from flashinfer import gelu_and_mul else: from freetoken.kernel.triton.activation import gelu_and_mul return gelu_and_mul(x, out=out) def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None): """tanh-approximate GELU gate (`gelu_pytorch_tanh`) followed by elementwise mul.""" if _use_flashinfer(x, out): from flashinfer import gelu_tanh_and_mul else: from freetoken.kernel.triton.activation import gelu_tanh_and_mul return gelu_tanh_and_mul(x, out=out) def swigluoai_and_mul( x: torch.Tensor, out: torch.Tensor | None = None, *, alpha: float = 1.702, limit: float = 7.0, ): """SwiGLU-OAI (gpt-oss / MiniMax-M3 ``swigluoai``) over UNINTERLEAVED halves (gate ``x[..., :d]``, up ``x[..., d:]``): ``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``. Always the in-repo Triton kernel (flashinfer ships no clamped-swiglu *_and_mul).""" from freetoken.kernel.triton.activation import swigluoai_and_mul return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit) def swiglu_clamp_and_mul( x, out=None, *, alpha: float = 1.0, limit: float = 10.0 ): """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``.""" from freetoken.kernel.triton.activation import swiglu_clamp_and_mul return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit) GATED_ACTIVATIONS = ("silu", "gelu", "gelu_tanh", "swigluoai", "swiglu_clamp") def gated_act_and_mul(activation: str, x, out, *, alpha: float = 1.0, limit: float = float("inf")): """The epilogue between the two expert GEMMs: ``activation`` over uninterleaved [gate; up] halves into ``out``.""" if activation == "silu": return silu_and_mul(x, out) if activation == "gelu": return gelu_and_mul(x, out) if activation == "gelu_tanh": return gelu_tanh_and_mul(x, out) if activation == "swigluoai": return swigluoai_and_mul(x, out, alpha=alpha, limit=limit) if activation == "swiglu_clamp": return swiglu_clamp_and_mul(x, out, alpha=alpha, limit=limit) raise ValueError(f"no gated activation {activation!r}; known: {GATED_ACTIVATIONS}") __all__ = [ "silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul", "swiglu_clamp_and_mul", "gated_act_and_mul", "GATED_ACTIVATIONS", ]