from typing import Tuple import torch import torch.nn.functional as F from .base import BaseOP class RMSNorm(BaseOP): def __init__(self, size: int, eps: float) -> None: from freetoken.kernel.backend import is_flashinfer_installed if is_flashinfer_installed(): from flashinfer import rmsnorm else: from freetoken.kernel.triton.norm import rmsnorm self.eps = eps self.weight = torch.empty(size) self.rmsnorm = rmsnorm def forward(self, x: torch.Tensor) -> torch.Tensor: return self.rmsnorm(x, self.weight, self.eps) def forward_inplace(self, x: torch.Tensor) -> None: self.rmsnorm(x, self.weight, self.eps, out=x) class GemmaRMSNorm(BaseOP): """Gemma4-style RMSNorm backed directly by sgl_kernel. Gemma4 scales by the raw checkpoint weight. ``with_scale=False`` uses a runtime ones vector that is intentionally not part of ``state_dict``. """ def __init__(self, size: int, eps: float, with_scale: bool = True) -> None: from freetoken.kernel.backend import is_sgl_kernel_installed if is_sgl_kernel_installed(): from sgl_kernel import fused_add_rmsnorm, rmsnorm else: from freetoken.kernel.triton.norm import fused_add_rmsnorm, rmsnorm self.eps = eps self.size = size self.with_scale = with_scale self.rmsnorm = rmsnorm self.fused_add_rmsnorm = fused_add_rmsnorm if with_scale: self.weight = torch.empty(size) else: self._ones_weight: torch.Tensor | None = None def _kernel_weight(self, x: torch.Tensor) -> torch.Tensor: if self.with_scale: return self.weight if self._ones_weight is None: self._ones_weight = torch.ones(self.size, device=x.device, dtype=x.dtype) return self._ones_weight def forward(self, x: torch.Tensor) -> torch.Tensor: if x.dim() == 2: return self.rmsnorm(x, self._kernel_weight(x), self.eps) original_shape = x.shape out = self.rmsnorm( x.contiguous().reshape(-1, original_shape[-1]), self._kernel_weight(x), self.eps, ) return out.reshape(original_shape) def forward_add_residual( self, x: torch.Tensor, residual: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: self.fused_add_rmsnorm(x, residual, self._kernel_weight(x), self.eps) return x, residual class GemmaPlusOneRMSNorm(BaseOP): """(1 + w)-scaled RMSNorm (Gemma semantics: the checkpoint stores ``scale - 1`` and the effective multiplier is ``1 + weight``, added in fp32 at runtime -- never folded into the bf16 weight, which would round away the precision the format exists to keep). Used by MiniMax-M3 (``use_gemma_norm``) for the decoder layernorms, the per-head q/k norms, and the indexer q/k norms. Per-head 3D inputs are collapsed to 2D before the kernel call: flashinfer's ``gemma_rmsnorm`` CUDA binding is CHECK_DIM(2) (it rejects 3D outright on wheels without the CuTe path), and the per-head weight makes the 2D view exactly equivalent. The M3 call sites pass contiguous buffers, so the views are free; a non-contiguous 3D input is rejected rather than silently copied (an in-place norm on a copy would be dropped). """ def __init__(self, size: int, eps: float) -> None: from freetoken.kernel.backend import is_flashinfer_installed if is_flashinfer_installed(): from flashinfer.norm import gemma_rmsnorm else: from freetoken.kernel.triton.norm import gemma_rmsnorm self.eps = eps self.size = size self.weight = torch.empty(size) self.gemma_rmsnorm = gemma_rmsnorm def _flat(self, x: torch.Tensor) -> torch.Tensor: if x.dim() == 2: return x assert x.is_contiguous(), "per-head gemma norm needs a contiguous buffer" return x.view(-1, self.size) def forward(self, x: torch.Tensor) -> torch.Tensor: return self.gemma_rmsnorm(self._flat(x), self.weight, self.eps).view(x.shape) def forward_inplace(self, x: torch.Tensor) -> None: flat = self._flat(x) self.gemma_rmsnorm(flat, self.weight, self.eps, out=flat) class GemmaPlusOneRMSNormFused(BaseOP): """(1 + w)-scaled RMSNorm with the fused-add-residual API of ``RMSNormFused`` (Gemma semantics, see :class:`GemmaPlusOneRMSNorm`). Drop-in for the decoder layernorm seam: ``forward(x, residual)`` returns ``(normed, residual)``.""" def __init__(self, size: int, eps: float) -> None: from freetoken.kernel.backend import is_flashinfer_installed if is_flashinfer_installed(): from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm else: from freetoken.kernel.triton.norm import ( gemma_fused_add_rmsnorm, gemma_rmsnorm, ) self.eps = eps self.weight = torch.empty(size) self.gemma_rmsnorm = gemma_rmsnorm self.gemma_fused_add_rmsnorm = gemma_fused_add_rmsnorm def forward( self, x: torch.Tensor, residual: torch.Tensor | None = None ) -> Tuple[torch.Tensor, torch.Tensor]: if residual is None: return self.gemma_rmsnorm(x, self.weight, self.eps), x self.gemma_fused_add_rmsnorm(x, residual, self.weight, self.eps) return x, residual class RMSNormFused(BaseOP): def __init__(self, size: int, eps: float) -> None: from freetoken.kernel.backend import is_flashinfer_installed if is_flashinfer_installed(): from flashinfer import fused_add_rmsnorm, rmsnorm else: from freetoken.kernel.triton.norm import fused_add_rmsnorm, rmsnorm self.eps = eps self.weight = torch.empty(size) self.rmsnorm = rmsnorm self.fused_add_rmsnorm = fused_add_rmsnorm def forward( self, x: torch.Tensor, residual: torch.Tensor | None = None ) -> Tuple[torch.Tensor, torch.Tensor]: if residual is None: return self.rmsnorm(x, self.weight, self.eps), x self.fused_add_rmsnorm(x, residual, self.weight, self.eps) return x, residual _GATE_ACTIVATIONS = ("silu", "swish", "sigmoid") class GatedRMSNorm(BaseOP): """``rmsnorm(x) * activation(z)`` in one fla kernel: the GDN / KDA output norm.""" def __init__(self, size: int, eps: float, activation: str = "silu") -> None: # rms_norm_gated drops the gate entirely (no error) for a name it does not know assert activation in _GATE_ACTIVATIONS, f"unsupported gate activation {activation!r}" self.weight = torch.empty(size) self.eps = eps self.activation = activation def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: from freetoken.kernel.fla import rms_norm_gated return rms_norm_gated( x=x, weight=self.weight, bias=None, z=z, eps=self.eps, is_rms_norm=True, norm_before_gate=True, activation=self.activation, ) class LayerNorm(BaseOP): """LayerNorm with bias on torch's fused kernel; the decoders use the RMSNorm family.""" def __init__(self, size: int, eps: float) -> None: self.eps = eps self.weight = torch.empty(size) self.bias = torch.empty(size) def forward(self, x: torch.Tensor) -> torch.Tensor: return F.layer_norm(x, (x.shape[-1],), self.weight, self.bias, self.eps)