import functools import json import os from pathlib import Path from typing import Dict, Tuple import torch from freetoken.utils import div_ceil def _torch_fused_topk( gating_output: torch.Tensor, topk: int, renormalize: bool, num_token_non_padded: torch.Tensor | None, ) -> Tuple[torch.Tensor, torch.Tensor]: """Pure-torch reference for the fused softmax router; tests compare the kernel against it. Softmax over all experts, select the top-k, and (when ``renormalize``) rescale the selected weights to sum to 1 -- the standard fused-MoE routing convention. """ probs = torch.softmax(gating_output.float(), dim=-1) topk_weights, topk_ids = torch.topk(probs, topk, dim=-1) if renormalize: topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) topk_ids = topk_ids.to(torch.int32) if num_token_non_padded is not None: indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device) topk_ids[indices >= num_token_non_padded, :] = -1 return topk_weights.contiguous(), topk_ids.contiguous() def fused_topk( hidden_states: torch.Tensor, gating_output: torch.Tensor, topk: int, renormalize: bool, num_token_non_padded: torch.Tensor | None = None, ) -> Tuple[torch.Tensor, torch.Tensor]: assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" from freetoken.kernel.triton.moe_router import fused_topk_softmax return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded) def moe_align_block_size( topk_ids: torch.Tensor, block_size: int, num_experts: int ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Aligns the token distribution across experts to be compatible with block size for matrix multiplication. Parameters: - topk_ids: A tensor of shape [total_tokens, top_k] representing the top-k expert indices for each token. - block_size: The block size used in block matrix multiplication. - num_experts: The total number of experts. Returns: - sorted_token_ids: A tensor containing the sorted token indices according to their allocated expert. - expert_ids: A tensor indicating the assigned expert index for each block. - num_tokens_post_padded: The total number of tokens after padding, ensuring divisibility by block_size. This function pads the number of tokens that each expert needs to process so that it is divisible by block_size. Padding ensures that during block matrix multiplication, the dimensions align correctly. Example: Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], block_size = 4, and num_experts = 4: - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, with each expert needing to process 3 tokens. - As block_size is 4, we pad 1 token for each expert. - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3]. - Then append padding tokens [12, 12, 12, 12] for each block. - After sorting by expert index, we obtain token_ids [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12]. Tokens 12 are non-existent (padding) and are ignored in the subsequent matrix multiplication. - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations. """ from freetoken.kernel.backend import is_rocm, is_sgl_kernel_installed if is_rocm(): import triton from packaging.version import Version # FIXME: remove once Triton 3.8 is unsupported. Its AMD backend folds compares on tl.histogram results to false. # https://github.com/triton-lang/triton/pull/11246 if Version(triton.__version__).release[:2] == (3, 8): from freetoken.kernel import moe_align_block_size_triton return moe_align_block_size_triton(topk_ids, block_size, num_experts) if not is_sgl_kernel_installed(): from freetoken.kernel.triton.moe_align import ( moe_align_block_size as triton_moe_align_block_size, ) return triton_moe_align_block_size(topk_ids, block_size, num_experts) from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size if topk_ids.numel() < num_experts + 1: max_num_tokens_padded = topk_ids.numel() * block_size else: max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device) max_num_m_blocks = div_ceil(max_num_tokens_padded, block_size) expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device) num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=topk_ids.device) cumsum_buffer = torch.empty((num_experts + 2,), dtype=torch.int32, device=topk_ids.device) sgl_moe_align_block_size( topk_ids, num_experts + 1, block_size, sorted_ids, expert_ids, num_tokens_post_pad, cumsum_buffer, True, ) return sorted_ids, expert_ids, num_tokens_post_pad def get_default_config( M: int, E: int, N: int, K: int, topk: int, ) -> Dict[str, int]: config = { "BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 8, } if M <= E: config = { "BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, } return config @functools.lru_cache(maxsize=32) def _get_tuned_moe_configs( num_experts: int, config_n: int, device_name: str, triton_version: str, ) -> Dict[int, Dict[str, int]] | None: file_name = f"E={num_experts},N={config_n},device_name={device_name}.json" version_dir = f"triton_{triton_version.replace('.', '_')}" config_roots = [] if env_dir := os.environ.get("FREETOKEN_MOE_CONFIG_DIR"): config_roots.append(Path(env_dir)) config_roots.append(Path(__file__).with_name("configs")) for root in config_roots: path = root / version_dir / file_name if not path.exists(): continue with path.open() as f: configs = json.load(f) return {int(batch_size): config for batch_size, config in configs.items()} return None def try_get_optimal_moe_config( w1_shape: Tuple[int, ...], w2_shape: Tuple[int, ...], top_k: int, M: int, ) -> Dict[str, int]: E, N, config_n = w2_shape config = None if torch.cuda.is_available(): import triton configs = _get_tuned_moe_configs( E, config_n, torch.cuda.get_device_name().replace(" ", "_"), triton.__version__, ) if configs: config = dict(configs[min(configs.keys(), key=lambda batch: abs(batch - M))]) if config is None: config = get_default_config(M, E, N, w1_shape[2], top_k) return config def fused_experts_impl( hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, activation: str = "silu", apply_router_weight_on_input: bool = False, act_alpha: float = 1.0, act_limit: float = float("inf"), w1_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, ) -> torch.Tensor: """Returns ``hidden_states`` itself, overwritten with the routed output. A caller that still needs the input afterwards (a shared expert, a residual) must read it BEFORE this call or pass a copy. ``fused_experts_decode_impl`` allocates instead, so the contract is not shared; the resident bf16 path routes decode through here too.""" from freetoken.kernel import fused_moe_kernel_triton, moe_sum_reduce_triton from freetoken.layers import gated_act_and_mul padded_size = 0 assert hidden_states.shape[1] == w1.shape[2] - padded_size, "Hidden size mismatch" assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" assert w1.is_contiguous(), "Expert weights1 must be contiguous" assert w2.is_contiguous(), "Expert weights2 must be contiguous" assert hidden_states.dtype in [torch.float32, torch.float16, torch.bfloat16] # both bias adds assume the router weight scales the expert output assert (w1_bias is None and w2_bias is None) or not apply_router_weight_on_input, "per-expert biases need the router weight on the output" num_tokens, _ = hidden_states.shape E, N, _ = w1.shape M = num_tokens get_config_func = functools.partial( try_get_optimal_moe_config, w1.shape, (w2.shape[0], w2.shape[1], w2.shape[2] - padded_size), topk_ids.shape[1], ) config = get_config_func(M) cache = torch.empty( M * topk_ids.shape[1] * max(N, w2.shape[1]), device=hidden_states.device, dtype=hidden_states.dtype, ) intermediate_cache1 = cache[: M * topk_ids.shape[1] * N].view( (M, topk_ids.shape[1], N), ) intermediate_cache2 = torch.empty( (M * topk_ids.shape[1], N // 2), device=hidden_states.device, dtype=hidden_states.dtype, ) intermediate_cache3 = cache[: M * topk_ids.shape[1] * w2.shape[1]].view( (M, topk_ids.shape[1], w2.shape[1]), ) compute_type = hidden_states.dtype out_hidden_states = hidden_states curr_hidden_states = hidden_states tokens_num, _ = curr_hidden_states.shape begin_token_idx, end_token_idx = 0, num_tokens intermediate_cache1 = intermediate_cache1[:tokens_num] intermediate_cache2 = intermediate_cache2[: tokens_num * topk_ids.shape[1]] intermediate_cache3 = intermediate_cache3[:tokens_num] config = get_config_func(tokens_num) curr_topk_ids = topk_ids[begin_token_idx:end_token_idx] curr_topk_weights = topk_weights[begin_token_idx:end_token_idx] sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( curr_topk_ids, config["BLOCK_SIZE_M"], E ) fused_moe_kernel_triton( curr_hidden_states, w1, intermediate_cache1, curr_topk_weights, curr_topk_ids, sorted_token_ids, expert_ids, num_tokens_post_padded, apply_router_weight_on_input, topk_ids.shape[1], config, compute_type=compute_type, ) if w1_bias is not None: # gpt-oss: the expert's gate/up bias is added before the gated activation intermediate_cache1 += w1_bias[curr_topk_ids] gated_act_and_mul(activation, intermediate_cache1.view(-1, N), intermediate_cache2, alpha=act_alpha, limit=act_limit) fused_moe_kernel_triton( intermediate_cache2, w2, (intermediate_cache3), curr_topk_weights, curr_topk_ids, sorted_token_ids, expert_ids, num_tokens_post_padded, not apply_router_weight_on_input, 1, config, compute_type=compute_type, ) moe_sum_reduce_triton( intermediate_cache3, out_hidden_states[begin_token_idx:end_token_idx], ) if w2_bias is not None: # the down bias is weighted like the expert's output: sum_k w_k * bias_k out_hidden_states[begin_token_idx:end_token_idx] += (curr_topk_weights.unsqueeze(-1) * w2_bias[curr_topk_ids]).sum(1) return out_hidden_states def fused_experts_decode_impl( hidden_states: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, activation: str = "silu", apply_router_weight_on_input: bool = False, act_alpha: float = 1.0, act_limit: float = float("inf"), w1_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, ) -> torch.Tensor: from freetoken.kernel import fused_moe_decode_kernel_triton, moe_sum_reduce_triton from freetoken.layers import gated_act_and_mul assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch" assert w1.shape[0] == w2.shape[0], "Expert cache size mismatch" assert w1.shape[1] == 2 * w2.shape[2], "Intermediate size mismatch" assert w2.shape[1] == hidden_states.shape[1], "Output hidden size mismatch" assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" assert w1.is_contiguous(), "Expert weights1 must be contiguous" assert w2.is_contiguous(), "Expert weights2 must be contiguous" assert topk_weights.is_contiguous(), "topk_weights must be contiguous" assert topk_ids.is_contiguous(), "topk_ids must be contiguous" assert hidden_states.dtype in [torch.float16, torch.bfloat16] assert w1.dtype == hidden_states.dtype and w2.dtype == hidden_states.dtype assert topk_weights.dtype == torch.float32 assert topk_ids.dtype == torch.int32 # both bias adds assume the router weight scales the expert output assert (w1_bias is None and w2_bias is None) or not apply_router_weight_on_input, "per-expert biases need the router weight on the output" M, _ = hidden_states.shape top_k = topk_ids.shape[1] gate_up_dim = w1.shape[1] intermediate_size = gate_up_dim // 2 config = { "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64, "num_warps": 8, } intermediate_cache1 = torch.empty( (M, top_k, gate_up_dim), device=hidden_states.device, dtype=hidden_states.dtype, ) fused_moe_decode_kernel_triton( hidden_states, w1, intermediate_cache1, topk_weights, topk_ids, apply_router_weight_on_input, False, config, compute_type=hidden_states.dtype, ) if w1_bias is not None: intermediate_cache1 += w1_bias[topk_ids] intermediate_cache2 = torch.empty( (M * top_k, intermediate_size), device=hidden_states.device, dtype=hidden_states.dtype, ) gated_act_and_mul(activation, intermediate_cache1.view(-1, gate_up_dim), intermediate_cache2, alpha=act_alpha, limit=act_limit) intermediate_cache3 = torch.empty( (M, top_k, w2.shape[1]), device=hidden_states.device, dtype=hidden_states.dtype, ) fused_moe_decode_kernel_triton( intermediate_cache2, w2, intermediate_cache3, topk_weights, topk_ids, not apply_router_weight_on_input, True, config, compute_type=hidden_states.dtype, ) out_hidden_states = torch.empty_like(hidden_states) moe_sum_reduce_triton(intermediate_cache3, out_hidden_states) if w2_bias is not None: out_hidden_states += (topk_weights.unsqueeze(-1) * w2_bias[topk_ids]).sum(1) return out_hidden_states