mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
Serve Qwen3.8-Flash-Next (HF model_type qwen4_exp) text-only: 36 GDN + 12 QSA compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE n-gram embedding layer backed by a 47.7 GiB pinned-host table with UVA gather, and 512 NVFP4 / block-fp8 routed experts (top-10) plus a gated shared expert. - attention: qsa_sparse backend (AttnType.QSA) over QSAKVCache -- paged GQA K/V, a 1/ratio compressed index-key slab shadowing the KV pages, and a per-request pending ring sized from index_ratio - kvcache: declarative slot-sibling states (ModelConfig.slot_states) on LinearStatePool carry the PLE conv history and n-gram context through the hybrid-radix snapshot/COW lifecycle - scheduler: hybrid prefill chunks align to the page size so snapshots land on donatable boundaries - kernels: triton kernels adapted from vLLM/SGLang (hc, qsa, ple gather, moe router / shared gate) plus an original radix block top-k; int64 row addressing throughout - moe: non-power-of-2 top-k router, deep-K marlin decode config, one fp8 scale-bank padding rule shared with the AOT row table - engine: the PLE table load reserves its pinned bytes from the pin budget before the expert banks plan their residency
138 lines
3.9 KiB
Python
138 lines
3.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
|
|
# Adapted from SGLang (kernels/ops/elementwise.py, ``_fused_gate_sigmoid_mul_add``)
|
|
"""Gated shared-expert epilogue: the gate reduction and the sigmoid-mul-add.
|
|
|
|
The routed experts may write into ``hidden_states`` in place, so the gate reduction runs
|
|
before them and the mul-add after.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
|
|
import torch
|
|
import triton
|
|
import triton.language as tl
|
|
|
|
from freetoken.utils.arch import is_sm90_supported
|
|
|
|
|
|
@functools.cache
|
|
def _pdl_supported() -> bool:
|
|
return is_sm90_supported()
|
|
|
|
|
|
def _reduction_warps(hidden_dim: int, num_tokens: int) -> int:
|
|
warps = max(min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), 32), 4)
|
|
return min(warps, 8) if num_tokens >= 1024 else warps
|
|
|
|
|
|
@triton.jit
|
|
def _gate_sigmoid_kernel(
|
|
hidden_ptr,
|
|
weight_ptr,
|
|
gate_ptr,
|
|
stride_h,
|
|
HIDDEN: tl.constexpr,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
launch_pdl: tl.constexpr,
|
|
) -> None:
|
|
# row * stride can overflow int32 for large token counts.
|
|
row = tl.program_id(0).to(tl.int64)
|
|
offs = tl.arange(0, BLOCK_SIZE)
|
|
mask = offs < HIDDEN
|
|
|
|
w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32)
|
|
|
|
if launch_pdl:
|
|
tl.extra.cuda.gdc_wait()
|
|
|
|
h = tl.load(hidden_ptr + row * stride_h + offs, mask=mask, other=0.0).to(tl.float32)
|
|
|
|
if launch_pdl:
|
|
tl.extra.cuda.gdc_launch_dependents()
|
|
|
|
tl.store(gate_ptr + row, tl.sigmoid(tl.sum(h * w, axis=0)))
|
|
|
|
|
|
def shared_gate_sigmoid(hidden_states: torch.Tensor, gate_weight: torch.Tensor) -> torch.Tensor:
|
|
"""Per-token ``sigmoid(hidden_states @ gate_weight)`` as fp32 [num_tokens]."""
|
|
num_tokens, hidden_dim = hidden_states.shape
|
|
assert hidden_states.stride(1) == 1, "shared gate requires unit inner stride"
|
|
assert gate_weight.shape == (hidden_dim,) and gate_weight.is_contiguous()
|
|
|
|
gate = torch.empty(num_tokens, dtype=torch.float32, device=hidden_states.device)
|
|
_gate_sigmoid_kernel[(num_tokens,)](
|
|
hidden_states,
|
|
gate_weight,
|
|
gate,
|
|
hidden_states.stride(0),
|
|
HIDDEN=hidden_dim,
|
|
BLOCK_SIZE=triton.next_power_of_2(hidden_dim),
|
|
launch_pdl=_pdl_supported(),
|
|
num_warps=_reduction_warps(hidden_dim, num_tokens),
|
|
)
|
|
return gate
|
|
|
|
|
|
@triton.jit
|
|
def _gate_mul_add_kernel(
|
|
routed_ptr,
|
|
shared_ptr,
|
|
gate_ptr,
|
|
out_ptr,
|
|
stride_r,
|
|
stride_s,
|
|
stride_o,
|
|
HIDDEN: tl.constexpr,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
launch_pdl: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0).to(tl.int64)
|
|
block = tl.program_id(1)
|
|
offs = block * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
|
mask = offs < HIDDEN
|
|
|
|
if launch_pdl:
|
|
tl.extra.cuda.gdc_wait()
|
|
|
|
gate = tl.load(gate_ptr + row)
|
|
routed = tl.load(routed_ptr + row * stride_r + offs, mask=mask, other=0.0).to(tl.float32)
|
|
shared = tl.load(shared_ptr + row * stride_s + offs, mask=mask, other=0.0).to(tl.float32)
|
|
|
|
if launch_pdl:
|
|
tl.extra.cuda.gdc_launch_dependents()
|
|
|
|
tl.store(out_ptr + row * stride_o + offs, routed + gate * shared, mask=mask)
|
|
|
|
|
|
def shared_gate_mul_add(
|
|
routed: torch.Tensor, shared: torch.Tensor, gate: torch.Tensor
|
|
) -> torch.Tensor:
|
|
"""``routed + gate[:, None] * shared`` into a fresh tensor."""
|
|
num_tokens, hidden_dim = routed.shape
|
|
assert shared.shape == routed.shape
|
|
assert routed.stride(1) == 1 and shared.stride(1) == 1
|
|
assert gate.shape == (num_tokens,)
|
|
|
|
out = torch.empty_like(routed)
|
|
block_size = min(triton.next_power_of_2(hidden_dim), 2048)
|
|
_gate_mul_add_kernel[(num_tokens, triton.cdiv(hidden_dim, block_size))](
|
|
routed,
|
|
shared,
|
|
gate,
|
|
out,
|
|
routed.stride(0),
|
|
shared.stride(0),
|
|
out.stride(0),
|
|
HIDDEN=hidden_dim,
|
|
BLOCK_SIZE=block_size,
|
|
launch_pdl=_pdl_supported(),
|
|
num_warps=4,
|
|
)
|
|
return out
|
|
|
|
|
|
__all__ = ["shared_gate_mul_add", "shared_gate_sigmoid"]
|