Files
Mukkesh Ganesh ea2916d31c feat(gguf): serve gpt-oss gguf checkpoints on gpt_oss (#838)
* feat(gguf): serve gpt-oss gguf checkpoints on gpt_oss

* fix(gguf): refuse missing gpt-oss bias tensors, honor swa pattern, scale nvfp4 before bias

* fix(gguf): honor gpt-oss yarn truncate, gate the bf16 float cast, scale nvfp4 before the down bias
2026-11-02 08:58:07 -08:00

411 lines
15 KiB
Python

import functools
import json
import os
from pathlib import Path
from typing import Dict, Tuple
import torch
from freetoken.utils import div_ceil
def _torch_fused_topk(
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
num_token_non_padded: torch.Tensor | None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Pure-torch reference for the fused softmax router; tests compare the kernel against it.
Softmax over all experts, select the top-k, and (when ``renormalize``) rescale the
selected weights to sum to 1 -- the standard fused-MoE routing convention.
"""
probs = torch.softmax(gating_output.float(), dim=-1)
topk_weights, topk_ids = torch.topk(probs, topk, dim=-1)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
topk_ids = topk_ids.to(torch.int32)
if num_token_non_padded is not None:
indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device)
topk_ids[indices >= num_token_non_padded, :] = -1
return topk_weights.contiguous(), topk_ids.contiguous()
def fused_topk(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
num_token_non_padded: torch.Tensor | None = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
from freetoken.kernel.triton.moe_router import fused_topk_softmax
return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)
def moe_align_block_size(
topk_ids: torch.Tensor, block_size: int, num_experts: int
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns the token distribution across experts to be compatible with block
size for matrix multiplication.
Parameters:
- topk_ids: A tensor of shape [total_tokens, top_k] representing the
top-k expert indices for each token.
- block_size: The block size used in block matrix multiplication.
- num_experts: The total number of experts.
Returns:
- sorted_token_ids: A tensor containing the sorted token indices according
to their allocated expert.
- expert_ids: A tensor indicating the assigned expert index for each block.
- num_tokens_post_padded: The total number of tokens after padding,
ensuring divisibility by block_size.
This function pads the number of tokens that each expert needs to process
so that it is divisible by block_size.
Padding ensures that during block matrix multiplication, the dimensions
align correctly.
Example:
Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]],
block_size = 4, and num_experts = 4:
- We initially have 12 tokens (after repeating 'top_k' times) and 4 experts,
with each expert needing to process 3 tokens.
- As block_size is 4, we pad 1 token for each expert.
- First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3].
- Then append padding tokens [12, 12, 12, 12] for each block.
- After sorting by expert index, we obtain token_ids
[3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
Tokens 12 are non-existent (padding) and are ignored in
the subsequent matrix multiplication.
- The padding ensures that the total number of tokens is now divisible
by block_size for proper block matrix operations.
"""
from freetoken.kernel.backend import is_rocm, is_sgl_kernel_installed
if is_rocm():
import triton
from packaging.version import Version
# FIXME: remove once Triton 3.8 is unsupported. Its AMD backend folds compares on tl.histogram results to false.
# https://github.com/triton-lang/triton/pull/11246
if Version(triton.__version__).release[:2] == (3, 8):
from freetoken.kernel import moe_align_block_size_triton
return moe_align_block_size_triton(topk_ids, block_size, num_experts)
if not is_sgl_kernel_installed():
from freetoken.kernel.triton.moe_align import (
moe_align_block_size as triton_moe_align_block_size,
)
return triton_moe_align_block_size(topk_ids, block_size, num_experts)
from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded = topk_ids.numel() * block_size
else:
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device)
max_num_m_blocks = div_ceil(max_num_tokens_padded, block_size)
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device)
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=topk_ids.device)
cumsum_buffer = torch.empty((num_experts + 2,), dtype=torch.int32, device=topk_ids.device)
sgl_moe_align_block_size(
topk_ids,
num_experts + 1,
block_size,
sorted_ids,
expert_ids,
num_tokens_post_pad,
cumsum_buffer,
True,
)
return sorted_ids, expert_ids, num_tokens_post_pad
def get_default_config(
M: int,
E: int,
N: int,
K: int,
topk: int,
) -> Dict[str, int]:
config = {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
}
if M <= E:
config = {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
}
return config
@functools.lru_cache(maxsize=32)
def _get_tuned_moe_configs(
num_experts: int,
config_n: int,
device_name: str,
triton_version: str,
) -> Dict[int, Dict[str, int]] | None:
file_name = f"E={num_experts},N={config_n},device_name={device_name}.json"
version_dir = f"triton_{triton_version.replace('.', '_')}"
config_roots = []
if env_dir := os.environ.get("FREETOKEN_MOE_CONFIG_DIR"):
config_roots.append(Path(env_dir))
config_roots.append(Path(__file__).with_name("configs"))
for root in config_roots:
path = root / version_dir / file_name
if not path.exists():
continue
with path.open() as f:
configs = json.load(f)
return {int(batch_size): config for batch_size, config in configs.items()}
return None
def try_get_optimal_moe_config(
w1_shape: Tuple[int, ...],
w2_shape: Tuple[int, ...],
top_k: int,
M: int,
) -> Dict[str, int]:
E, N, config_n = w2_shape
config = None
if torch.cuda.is_available():
import triton
configs = _get_tuned_moe_configs(
E,
config_n,
torch.cuda.get_device_name().replace(" ", "_"),
triton.__version__,
)
if configs:
config = dict(configs[min(configs.keys(), key=lambda batch: abs(batch - M))])
if config is None:
config = get_default_config(M, E, N, w1_shape[2], top_k)
return config
def fused_experts_impl(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
act_alpha: float = 1.0,
act_limit: float = float("inf"),
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Returns ``hidden_states`` itself, overwritten with the routed output. A caller that
still needs the input afterwards (a shared expert, a residual) must read it BEFORE this
call or pass a copy. ``fused_experts_decode_impl`` allocates instead, so the contract is
not shared; the resident bf16 path routes decode through here too."""
from freetoken.kernel import fused_moe_kernel_triton, moe_sum_reduce_triton
from freetoken.layers import gated_act_and_mul
padded_size = 0
assert hidden_states.shape[1] == w1.shape[2] - padded_size, "Hidden size mismatch"
assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
assert w1.is_contiguous(), "Expert weights1 must be contiguous"
assert w2.is_contiguous(), "Expert weights2 must be contiguous"
assert hidden_states.dtype in [torch.float32, torch.float16, torch.bfloat16]
# both bias adds assume the router weight scales the expert output
assert (w1_bias is None and w2_bias is None) or not apply_router_weight_on_input, "per-expert biases need the router weight on the output"
num_tokens, _ = hidden_states.shape
E, N, _ = w1.shape
M = num_tokens
get_config_func = functools.partial(
try_get_optimal_moe_config,
w1.shape,
(w2.shape[0], w2.shape[1], w2.shape[2] - padded_size),
topk_ids.shape[1],
)
config = get_config_func(M)
cache = torch.empty(
M * topk_ids.shape[1] * max(N, w2.shape[1]),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = cache[: M * topk_ids.shape[1] * N].view(
(M, topk_ids.shape[1], N),
)
intermediate_cache2 = torch.empty(
(M * topk_ids.shape[1], N // 2),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache3 = cache[: M * topk_ids.shape[1] * w2.shape[1]].view(
(M, topk_ids.shape[1], w2.shape[1]),
)
compute_type = hidden_states.dtype
out_hidden_states = hidden_states
curr_hidden_states = hidden_states
tokens_num, _ = curr_hidden_states.shape
begin_token_idx, end_token_idx = 0, num_tokens
intermediate_cache1 = intermediate_cache1[:tokens_num]
intermediate_cache2 = intermediate_cache2[: tokens_num * topk_ids.shape[1]]
intermediate_cache3 = intermediate_cache3[:tokens_num]
config = get_config_func(tokens_num)
curr_topk_ids = topk_ids[begin_token_idx:end_token_idx]
curr_topk_weights = topk_weights[begin_token_idx:end_token_idx]
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
curr_topk_ids, config["BLOCK_SIZE_M"], E
)
fused_moe_kernel_triton(
curr_hidden_states,
w1,
intermediate_cache1,
curr_topk_weights,
curr_topk_ids,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
apply_router_weight_on_input,
topk_ids.shape[1],
config,
compute_type=compute_type,
)
if w1_bias is not None:
# gpt-oss: the expert's gate/up bias is added before the gated activation
intermediate_cache1 += w1_bias[curr_topk_ids]
gated_act_and_mul(activation, intermediate_cache1.view(-1, N), intermediate_cache2, alpha=act_alpha, limit=act_limit)
fused_moe_kernel_triton(
intermediate_cache2,
w2,
(intermediate_cache3),
curr_topk_weights,
curr_topk_ids,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input,
1,
config,
compute_type=compute_type,
)
moe_sum_reduce_triton(
intermediate_cache3,
out_hidden_states[begin_token_idx:end_token_idx],
)
if w2_bias is not None:
# the down bias is weighted like the expert's output: sum_k w_k * bias_k
out_hidden_states[begin_token_idx:end_token_idx] += (curr_topk_weights.unsqueeze(-1) * w2_bias[curr_topk_ids]).sum(1)
return out_hidden_states
def fused_experts_decode_impl(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
act_alpha: float = 1.0,
act_limit: float = float("inf"),
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> torch.Tensor:
from freetoken.kernel import fused_moe_decode_kernel_triton, moe_sum_reduce_triton
from freetoken.layers import gated_act_and_mul
assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
assert w1.shape[0] == w2.shape[0], "Expert cache size mismatch"
assert w1.shape[1] == 2 * w2.shape[2], "Intermediate size mismatch"
assert w2.shape[1] == hidden_states.shape[1], "Output hidden size mismatch"
assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
assert w1.is_contiguous(), "Expert weights1 must be contiguous"
assert w2.is_contiguous(), "Expert weights2 must be contiguous"
assert topk_weights.is_contiguous(), "topk_weights must be contiguous"
assert topk_ids.is_contiguous(), "topk_ids must be contiguous"
assert hidden_states.dtype in [torch.float16, torch.bfloat16]
assert w1.dtype == hidden_states.dtype and w2.dtype == hidden_states.dtype
assert topk_weights.dtype == torch.float32
assert topk_ids.dtype == torch.int32
# both bias adds assume the router weight scales the expert output
assert (w1_bias is None and w2_bias is None) or not apply_router_weight_on_input, "per-expert biases need the router weight on the output"
M, _ = hidden_states.shape
top_k = topk_ids.shape[1]
gate_up_dim = w1.shape[1]
intermediate_size = gate_up_dim // 2
config = {
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"num_warps": 8,
}
intermediate_cache1 = torch.empty(
(M, top_k, gate_up_dim),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
fused_moe_decode_kernel_triton(
hidden_states,
w1,
intermediate_cache1,
topk_weights,
topk_ids,
apply_router_weight_on_input,
False,
config,
compute_type=hidden_states.dtype,
)
if w1_bias is not None:
intermediate_cache1 += w1_bias[topk_ids]
intermediate_cache2 = torch.empty(
(M * top_k, intermediate_size),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
gated_act_and_mul(activation, intermediate_cache1.view(-1, gate_up_dim), intermediate_cache2, alpha=act_alpha, limit=act_limit)
intermediate_cache3 = torch.empty(
(M, top_k, w2.shape[1]),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
fused_moe_decode_kernel_triton(
intermediate_cache2,
w2,
intermediate_cache3,
topk_weights,
topk_ids,
not apply_router_weight_on_input,
True,
config,
compute_type=hidden_states.dtype,
)
out_hidden_states = torch.empty_like(hidden_states)
moe_sum_reduce_triton(intermediate_cache3, out_hidden_states)
if w2_bias is not None:
out_hidden_states += (topk_weights.unsqueeze(-1) * w2_bias[topk_ids]).sum(1)
return out_hidden_states