Files

274 lines
10 KiB
Python

"""Tiled Triton RoPE (provenance: rope<-lightllm).
Adapted from lightllm's rotary_emb kernel (models/llama/triton_kernel/rotary_emb.py):
grid = (cdiv(nnz, BLOCK_SEQ), cdiv(num_heads, BLOCK_HEAD)); each program processes a
BLOCK_SEQ x BLOCK_HEAD x (rotary_dim/2) tile. Changed vs upstream:
* upstream takes pre-gathered cos/sin (nnz, dim/2); FreeToken passes a
cos_sin_cache (max_pos, rotary_dim) plus positions -> we gather
cache[positions[token]] inside the kernel (cos = first half, sin = second
half over rotary_dim).
* Q and K are rotated in a single kernel launch (one grid over max head
count, K stores masked past HEAD_K) to cut launch overhead for tiny decode
batches.
* rotation math in fp32 like the vendored/flashinfer kernel; is_neox and
interleave both supported as constexpr.
Optional pure-triton drop-in for
freetoken.kernel.rope.apply_rope_with_cos_sin_cache_inplace.
"""
from __future__ import annotations
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["nnz"])
def _rope_tiled(
Q, K, POS, CACHE,
stride_qbs, stride_qh, stride_qd,
stride_kbs, stride_kh, stride_kd,
nnz,
HEAD_Q, HEAD_K, rotary_dim, half,
HAS_K: tl.constexpr,
INTERLEAVE: tl.constexpr,
BLOCK_SEQ: tl.constexpr,
BLOCK_HEAD: tl.constexpr,
BLOCK_DHALF: tl.constexpr,
):
seq_pid = tl.program_id(0)
head_pid = tl.program_id(1)
seq_range = seq_pid * BLOCK_SEQ + tl.arange(0, BLOCK_SEQ) # (S,)
head_range = head_pid * BLOCK_HEAD + tl.arange(0, BLOCK_HEAD) # (H,)
d = tl.arange(0, BLOCK_DHALF) # (D,)
seq_mask = seq_range < nnz
dmask = d < half
pos = tl.load(POS + seq_range, mask=seq_mask, other=0).to(tl.int64) # (S,)
cache_row = CACHE + pos[:, None] * rotary_dim
cs_mask = seq_mask[:, None] & dmask[None, :]
cos = tl.load(cache_row + d[None, :], mask=cs_mask, other=0.0) # (S,D) fp32
sin = tl.load(cache_row + half + d[None, :], mask=cs_mask, other=0.0) # (S,D) fp32
cos = cos[:, None, :] # (S,1,D)
sin = sin[:, None, :]
if INTERLEAVE:
d0 = 2 * d
d1 = 2 * d + 1
else:
d0 = d
d1 = half + d
d0 = d0[None, None, :]
d1 = d1[None, None, :]
# --- Q ---
qmask = (seq_mask[:, None, None]
& (head_range[None, :, None] < HEAD_Q)
& dmask[None, None, :])
base_q = seq_range[:, None, None] * stride_qbs + head_range[None, :, None] * stride_qh
q0 = tl.load(Q + base_q + d0 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
q1 = tl.load(Q + base_q + d1 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
o0 = q0 * cos - q1 * sin
o1 = q1 * cos + q0 * sin
tl.store(Q + base_q + d0 * stride_qd, o0.to(Q.dtype.element_ty), mask=qmask)
tl.store(Q + base_q + d1 * stride_qd, o1.to(Q.dtype.element_ty), mask=qmask)
# --- K ---
if HAS_K:
kmask = (seq_mask[:, None, None]
& (head_range[None, :, None] < HEAD_K)
& dmask[None, None, :])
base_k = seq_range[:, None, None] * stride_kbs + head_range[None, :, None] * stride_kh
k0 = tl.load(K + base_k + d0 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
k1 = tl.load(K + base_k + d1 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
ok0 = k0 * cos - k1 * sin
ok1 = k1 * cos + k0 * sin
tl.store(K + base_k + d0 * stride_kd, ok0.to(K.dtype.element_ty), mask=kmask)
tl.store(K + base_k + d1 * stride_kd, ok1.to(K.dtype.element_ty), mask=kmask)
def apply_rope_with_cos_sin_cache_inplace(
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
head_size: int,
cos_sin_cache: torch.Tensor,
is_neox: bool = True,
) -> None:
"""Drop-in for the vendored op; rotates query/key in place."""
if str(cos_sin_cache.dtype) != "torch.float32":
raise ValueError("cos_sin_cache should be float32")
assert query.is_cuda and key.is_cuda and positions.is_cuda
assert cos_sin_cache.is_contiguous()
nnz = query.shape[0]
if nnz == 0:
return
rotary_dim = cos_sin_cache.shape[1]
half = rotary_dim // 2
block_dhalf = triton.next_power_of_2(half)
head_q = query.shape[1] // head_size
head_k = key.shape[1] // head_size
qv = query.view(nnz, head_q, head_size)
kv = key.view(nnz, head_k, head_size)
max_head = max(head_q, head_k)
grid = lambda META: (
triton.cdiv(nnz, META["BLOCK_SEQ"]),
triton.cdiv(max_head, META["BLOCK_HEAD"]),
)
# Fixed via H100 sweep (27-config grid; 16/1/w4 within 5% of the winner at
# every nnz 1..4096, faster than tuned on average).
_rope_tiled[grid](
qv, kv, positions, cos_sin_cache,
qv.stride(0), qv.stride(1), qv.stride(2),
kv.stride(0), kv.stride(1), kv.stride(2),
nnz, head_q, head_k, rotary_dim, half,
HAS_K=True,
INTERLEAVE=not is_neox,
BLOCK_DHALF=block_dhalf,
BLOCK_SEQ=16,
BLOCK_HEAD=1,
num_warps=4,
num_stages=1,
)
__all__ = ["apply_rope_with_cos_sin_cache_inplace"]
@triton.jit(do_not_specialize=["nnz"])
def _mrope_tiled(
Q, K, POS, CACHE, SEC,
stride_qbs, stride_qh, stride_qd,
stride_kbs, stride_kh, stride_kd,
nnz,
HEAD_Q, HEAD_K, rotary_dim, half,
BLOCK_SEQ: tl.constexpr,
BLOCK_HEAD: tl.constexpr,
BLOCK_DHALF: tl.constexpr,
):
# _rope_tiled with 3-axis positions: POS is [3, nnz], SEC[half] picks the axis each frequency slot reads
seq_pid = tl.program_id(0)
head_pid = tl.program_id(1)
seq_range = seq_pid * BLOCK_SEQ + tl.arange(0, BLOCK_SEQ)
head_range = head_pid * BLOCK_HEAD + tl.arange(0, BLOCK_HEAD)
d = tl.arange(0, BLOCK_DHALF)
seq_mask = seq_range < nnz
dmask = d < half
sec = tl.load(SEC + d, mask=dmask, other=0).to(tl.int64) # (D,) in {0,1,2}
pos_ptr = POS + sec[None, :] * nnz + seq_range[:, None]
cs_mask = seq_mask[:, None] & dmask[None, :]
pos = tl.load(pos_ptr, mask=cs_mask, other=0).to(tl.int64) # (S,D)
cache_row = CACHE + pos * rotary_dim
cos = tl.load(cache_row + d[None, :], mask=cs_mask, other=0.0)
sin = tl.load(cache_row + half + d[None, :], mask=cs_mask, other=0.0)
cos = cos[:, None, :]
sin = sin[:, None, :]
d0 = d[None, None, :]
d1 = (half + d)[None, None, :]
qmask = (seq_mask[:, None, None]
& (head_range[None, :, None] < HEAD_Q)
& dmask[None, None, :])
base_q = seq_range[:, None, None] * stride_qbs + head_range[None, :, None] * stride_qh
q0 = tl.load(Q + base_q + d0 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
q1 = tl.load(Q + base_q + d1 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
tl.store(Q + base_q + d0 * stride_qd, (q0 * cos - q1 * sin).to(Q.dtype.element_ty), mask=qmask)
tl.store(Q + base_q + d1 * stride_qd, (q1 * cos + q0 * sin).to(Q.dtype.element_ty), mask=qmask)
kmask = (seq_mask[:, None, None]
& (head_range[None, :, None] < HEAD_K)
& dmask[None, None, :])
base_k = seq_range[:, None, None] * stride_kbs + head_range[None, :, None] * stride_kh
k0 = tl.load(K + base_k + d0 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
k1 = tl.load(K + base_k + d1 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
tl.store(K + base_k + d0 * stride_kd, (k0 * cos - k1 * sin).to(K.dtype.element_ty), mask=kmask)
tl.store(K + base_k + d1 * stride_kd, (k1 * cos + k0 * sin).to(K.dtype.element_ty), mask=kmask)
def apply_mrope_with_cos_sin_cache_inplace(
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
head_size: int,
cos_sin_cache: torch.Tensor,
section_table: torch.Tensor,
) -> None:
"""3-axis rope: positions is [3, nnz], section_table [rotary_dim//2] maps each frequency slot to its axis row; NeoX layout only."""
if str(cos_sin_cache.dtype) != "torch.float32":
raise ValueError("cos_sin_cache should be float32")
assert positions.dim() == 2 and positions.shape[0] == 3
assert query.is_cuda and key.is_cuda and positions.is_cuda
assert cos_sin_cache.is_contiguous() and positions.is_contiguous()
nnz = query.shape[0]
if nnz == 0:
return
rotary_dim = cos_sin_cache.shape[1]
half = rotary_dim // 2
head_q = query.shape[1] // head_size
head_k = key.shape[1] // head_size
qv = query.view(nnz, head_q, head_size)
kv = key.view(nnz, head_k, head_size)
max_head = max(head_q, head_k)
grid = lambda META: (
triton.cdiv(nnz, META["BLOCK_SEQ"]),
triton.cdiv(max_head, META["BLOCK_HEAD"]),
)
_mrope_tiled[grid](
qv, kv, positions, cos_sin_cache, section_table,
qv.stride(0), qv.stride(1), qv.stride(2),
kv.stride(0), kv.stride(1), kv.stride(2),
nnz, head_q, head_k, rotary_dim, half,
BLOCK_DHALF=triton.next_power_of_2(half),
BLOCK_SEQ=16,
BLOCK_HEAD=1,
num_warps=4,
num_stages=1,
)
def apply_mrope_torch_fallback(
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
head_size: int,
cos_sin_cache: torch.Tensor,
section_table: torch.Tensor,
) -> None:
"""Pure-torch mrope (Triton unavailable); same in-place contract as the kernel."""
nnz = query.shape[0]
if nnz == 0:
return
rotary_dim = cos_sin_cache.shape[1]
half = rotary_dim // 2
pos = positions[section_table.long(), :].transpose(0, 1) # [nnz, half]
cs = cos_sin_cache[pos] # [nnz, half, 2*half] rows gathered per slot
idx = torch.arange(half, device=query.device)
cos = cs[:, idx, idx].unsqueeze(1).float()
sin = cs[:, idx, half + idx].unsqueeze(1).float()
for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):
v = t.view(nnz, heads, head_size)
lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()
# Compute both halves before writing: float() aliases FP32 inputs.
lo_new = lo * cos - hi * sin
hi_new = hi * cos + lo * sin
v[..., :half] = lo_new.to(v.dtype)
v[..., half:rotary_dim] = hi_new.to(v.dtype)
__all__ = ["apply_rope_with_cos_sin_cache_inplace", "apply_mrope_with_cos_sin_cache_inplace"]