mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* feat(spec): add the speculative verify infrastructure * feat(scheduler): run decode steps as speculative verify steps * chore(engine): mark the speculative decoding config fields with one comment * feat(server): add short aliases for the speculative decoding flags
212 lines
9.2 KiB
Python
212 lines
9.2 KiB
Python
"""GLM-5.3-Flash KDA (Kimi Delta Attention) op.
|
|
|
|
Per-channel gated delta rule over H=64 heads of D=128, with SEPARATE q/k/v short
|
|
convolutions (vs GDN's one fused conv), a low-rank forget gate (``f_a`` ->
|
|
``f_b`` -> raw per-channel logits; the bounded safe gate ``lower_bound *
|
|
sigmoid(exp(A_log) * (g + dt_bias))`` is computed inside the kernels), a
|
|
per-head sigmoid beta (``b_proj``), and a sigmoid-gated output RMSNorm
|
|
(``o_norm`` gated by ``g_b(g_a(x))``).
|
|
|
|
State lives in ``ctx.linear_state_pool`` exactly like GDN: the conv state is the
|
|
merged q|k|v stream (width 3*H*D == the pool's ``2*K*dk + V*dv``) and the
|
|
recurrent state is one [D, D] matrix per head, stored in the KERNEL's [V, K]
|
|
layout (coincides with the pool's [K, V] declaration because D_k == D_v; same
|
|
convention as GDN, see layers/linear_attn/gdn.py).
|
|
|
|
Kernels: ``fused_recurrent_kda`` decodes with in-kernel gate/beta/l2norm and
|
|
per-slot state read/write (slot 0 is its NULL sentinel == the pool's padding
|
|
slot); ``chunk_kda_with_fused_gate`` prefills from an explicitly gathered
|
|
initial state and returns the final state, which this op scatters back (the
|
|
kernel CLOBBERS its v buffer -- v here is an ephemeral conv output, so that is
|
|
free). Hybrid-radix track snapshots ride the per-chunk h (``return_h``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
import torch
|
|
from freetoken.core import get_global_ctx
|
|
from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
|
|
from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated
|
|
from freetoken.utils import nvtx_annotate
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.models.config import ModelConfig
|
|
|
|
|
|
class _DepthwiseConv1d(BaseOP):
|
|
"""Merged q|k|v depthwise conv weight ``[3*H*D, 1, K]`` (key ``conv1d.weight``;
|
|
the loader concatenates the checkpoint's q/k/v_conv1d along channels)."""
|
|
|
|
def __init__(self, conv_dim: int, kernel: int):
|
|
self.weight = torch.empty(conv_dim, 1, kernel)
|
|
|
|
|
|
class Glm5NextKDA(BaseOP):
|
|
"""KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request's
|
|
linear slot (``FLAMetadata.cache_indices``). Parameter names follow the
|
|
checkpoint modulo two load-time fusions (see weight.py): ``in_proj`` is
|
|
q|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated."""
|
|
|
|
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
|
|
args = config.glm5_args
|
|
self.layer_id = layer_id
|
|
self.num_heads = args.linear_num_heads
|
|
self.head_dim = args.linear_head_dim
|
|
self.proj_size = self.num_heads * self.head_dim # H * D
|
|
self.conv_dim = 3 * self.proj_size # merged q|k|v stream
|
|
self.conv_kernel_size = args.linear_conv_kernel_dim
|
|
self.lower_bound = args.linear_lower_bound
|
|
self.scale = self.head_dim**-0.5
|
|
|
|
p, h, d = self.proj_size, self.num_heads, self.head_dim
|
|
# one fused input GEMM over q|k|v|b|f_a|g_a
|
|
self._in_proj_split = [p, p, p, h, d, d]
|
|
self.in_proj = LinearColParallelMerged(
|
|
args.hidden_size, self._in_proj_split, has_bias=False,
|
|
quant_config=config.quant, prefix=f"{prefix}.in_proj",
|
|
)
|
|
# Low-rank gate up-projections (128 -> 8192): forget gate and output gate.
|
|
self.f_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.f_b_proj")
|
|
self.g_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.g_b_proj")
|
|
self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size)
|
|
# Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32).
|
|
# models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast.
|
|
self.A_log = torch.empty(h, dtype=torch.float32)
|
|
self.dt_bias = torch.empty(p, dtype=torch.float32)
|
|
self.o_norm = GatedRMSNorm(d, eps=args.norm_eps, activation="sigmoid")
|
|
self.o_proj = LinearReplicated(p, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.o_proj")
|
|
|
|
def _conv_weight(self) -> torch.Tensor:
|
|
return self.conv1d.weight.squeeze(1) # [conv_dim, kernel]
|
|
|
|
def _write_track_snapshot(self, pool, li, conv_in, h, fla) -> None:
|
|
"""Hybrid-radix: snapshot recurrent + conv state at the chunk-aligned track
|
|
boundary into a donatable pool slot (same contract as GDN, see
|
|
layers/linear_attn/gdn.py). h rows are the kernel's per-chunk [V, K] states --
|
|
a direct copy into the pool's [K, V] slots (D_k == D_v)."""
|
|
rec = pool.recurrent_states[li]
|
|
rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype))
|
|
cv = pool.conv_states[li]
|
|
conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous()
|
|
cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype))
|
|
|
|
@nvtx_annotate("KDA")
|
|
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
|
ctx = get_global_ctx()
|
|
batch = ctx.batch
|
|
pool = ctx.linear_state_pool
|
|
total = hidden_states.shape[0]
|
|
dtype = hidden_states.dtype
|
|
h, d, p = self.num_heads, self.head_dim, self.proj_size
|
|
|
|
fla = batch.fla_metadata
|
|
if fla is None:
|
|
from freetoken.attention.linear import build_fla_metadata
|
|
|
|
fla = build_fla_metadata(batch, hidden_states.device, pool)
|
|
batch.fla_metadata = fla
|
|
|
|
proj = self.in_proj.forward(hidden_states)
|
|
conv_in, b, f_a, g_a = torch.split(
|
|
proj, [self.conv_dim, h, d, d], dim=-1
|
|
)
|
|
g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D]
|
|
g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D]
|
|
li = pool.local_index(self.layer_id)
|
|
|
|
if batch.is_decode:
|
|
mixed = causal_conv1d_decode(
|
|
conv_in, pool.conv_states[li], self._conv_weight(), fla.cache_indices
|
|
)
|
|
bsz = mixed.shape[0]
|
|
q, k, v = (
|
|
t.reshape(1, bsz, h, d).to(dtype)
|
|
for t in torch.split(mixed, [p, p, p], dim=-1)
|
|
)
|
|
core_out, _ = _fused_recurrent(
|
|
q, k, v,
|
|
g=g1.view(1, bsz, h, d),
|
|
beta=b.view(1, bsz, h),
|
|
state_pool=pool.recurrent_states[li],
|
|
indices=fla.cache_indices,
|
|
cu_seqlens=fla.cu_seqlens,
|
|
a_log=self.A_log,
|
|
dt_bias=self.dt_bias,
|
|
lower_bound=self.lower_bound,
|
|
scale=self.scale,
|
|
)
|
|
else:
|
|
x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total]
|
|
mixed = causal_conv1d_varlen(
|
|
x, self._conv_weight(), pool.conv_states[li],
|
|
fla.cu_seqlens, fla.cache_indices, fla.has_initial_state,
|
|
).transpose(0, 1)
|
|
q, k, v = (
|
|
t.reshape(1, total, h, d).to(dtype)
|
|
for t in torch.split(mixed, [p, p, p], dim=-1)
|
|
)
|
|
# Fresh sequences start from a zeroed slot; then gather every request's
|
|
# initial state (the chunk kernel takes it dense, [N, H, D, D]).
|
|
rec = pool.recurrent_states[li]
|
|
if fla.fresh_state_indices is not None:
|
|
rec.index_fill_(0, fla.fresh_state_indices, 0.0)
|
|
slot_ids = fla.cache_indices.long()
|
|
initial = rec.index_select(0, slot_ids)
|
|
|
|
from freetoken.kernel.fla import chunk_kda_with_fused_gate
|
|
|
|
track = fla.track_dst is not None
|
|
result = chunk_kda_with_fused_gate(
|
|
q=q, k=k, v=v, # NOTE: v (ephemeral conv output) is clobbered
|
|
raw_g=g1.view(1, total, h, d),
|
|
beta=b.float().sigmoid().view(1, total, h),
|
|
A_log=self.A_log,
|
|
g_bias=self.dt_bias,
|
|
scale=self.scale,
|
|
initial_state=initial,
|
|
output_final_state=True,
|
|
use_qk_l2norm_in_kernel=True,
|
|
cu_seqlens=fla.cu_seqlens,
|
|
safe_gate=True,
|
|
lower_bound=self.lower_bound,
|
|
return_h=track,
|
|
)
|
|
if track:
|
|
core_out, final_state, chunk_h = result
|
|
self._write_track_snapshot(pool, li, conv_in, chunk_h, fla)
|
|
else:
|
|
core_out, final_state = result
|
|
rec.index_copy_(0, slot_ids, final_state.to(rec.dtype))
|
|
|
|
core_out = core_out.reshape(-1, d)
|
|
out = self.o_norm.forward(core_out, g2.reshape(-1, d)).reshape(total, -1)
|
|
return self.o_proj.forward(out.to(dtype))
|
|
|
|
|
|
def _fused_recurrent(
|
|
q, k, v, g, beta, state_pool, indices, cu_seqlens,
|
|
a_log, dt_bias, lower_bound, scale,
|
|
):
|
|
"""Decode via the vendored recurrent kernel: gate + beta-sigmoid + q/k l2norm
|
|
in-kernel, state read/written in place at ``indices`` (int32, 1 token/req)."""
|
|
from freetoken.kernel.fla import fused_recurrent_kda
|
|
|
|
return fused_recurrent_kda(
|
|
q=q, k=k, v=v, g=g, beta=beta,
|
|
scale=scale,
|
|
initial_state=state_pool,
|
|
use_qk_l2norm_in_kernel=True,
|
|
cu_seqlens=cu_seqlens,
|
|
ssm_state_indices=indices,
|
|
sigmoid_beta=True,
|
|
a_log=a_log,
|
|
g_bias=dt_bias,
|
|
compute_gate=True,
|
|
lower_bound=lower_bound,
|
|
)
|
|
|
|
|
|
__all__ = ["Glm5NextKDA"]
|