mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): add the speculative verify infrastructure * feat(scheduler): run decode steps as speculative verify steps * chore(engine): mark the speculative decoding config fields with one comment * feat(server): add short aliases for the speculative decoding flags
211 lines
10 KiB
Python
211 lines
10 KiB
Python
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass
|
||
from typing import TYPE_CHECKING, List
|
||
|
||
import torch
|
||
from freetoken.attention.base import AttentionSpec, BaseAttnBackend
|
||
|
||
if TYPE_CHECKING:
|
||
from freetoken.core import Batch
|
||
from freetoken.kvcache.linear_state_pool import LinearStatePool, LinearVerifyScratch
|
||
|
||
|
||
@dataclass
|
||
class FLAMetadata:
|
||
"""Per-forward GatedDeltaNet (flash-linear-attention) metadata, built once per
|
||
forward and shared by every GDN layer -- mirrors ``BaseAttnMetadata``. Replaces the
|
||
per-layer rebuilds the GDN op used to do (``cu_seqlens`` arange, per-request
|
||
``cache_indices``/``has_initial_state``), which were pageable, synchronous H2D copies
|
||
issued in each of the 30 GDN layers.
|
||
|
||
Fields:
|
||
cu_seqlens query indptr; decode = arange(bs+1) (1 token/req), prefill =
|
||
cumsum of extend_len. int32 on device.
|
||
cache_indices per-request recurrent/conv state slot (Req.linear_slot_idx, else
|
||
Req.table_idx). int32.
|
||
has_initial_state prefill only: whether each request continues a cached prefix
|
||
(cached_len > 0). None for decode (state always present).
|
||
fresh_state_indices prefill only: the state-pool slots whose sequence is fresh
|
||
(cached_len == 0) and must be zeroed before the chunk kernel
|
||
reads them in place. None if there are none / for decode.
|
||
"""
|
||
|
||
cu_seqlens: torch.Tensor
|
||
cache_indices: torch.Tensor
|
||
has_initial_state: torch.Tensor | None = None
|
||
fresh_state_indices: torch.Tensor | None = None
|
||
# verify only: the state pool's scratch the verify writes (row = batch row) and the commit replays
|
||
verify_scratch: "LinearVerifyScratch | None" = None
|
||
|
||
# --- hybrid-radix track-checkpoint (extra_buffer) fields; all None when not caching ---
|
||
# For each request crossing a chunk-aligned (×CHUNK) boundary this forward, snapshot its
|
||
# recurrent + conv state into a donatable pool slot, written on the forward stream by the
|
||
# GDN op (see GatedDeltaNet._write_track_snapshot). Built by the scheduler in P2;
|
||
# left None by build_fla_metadata so the existing path is unchanged.
|
||
track_dst: torch.Tensor | None = None # [nt] int64 dst pool slot per tracked req
|
||
track_h_row: torch.Tensor | None = None # [nt] int64 row into h (boh_i + aligned//CHUNK)
|
||
track_conv_src: torch.Tensor | None = None # [nt, kernel-1] int64 conv-input token positions
|
||
track_boundary_row: torch.Tensor | None = None # [nt] int64 forward-local row of the track boundary; states with their own left context (qwen4_exp PLE) derive their windows from it
|
||
|
||
|
||
def gdn_slot(req) -> int:
|
||
"""The request's GDN state slot: its hybrid-radix live slot (decoupled from table_idx) when allocated, else table_idx (naive GDN keeps the old keying)."""
|
||
return req.linear_slot_idx if req.linear_slot_idx is not None else req.table_idx
|
||
|
||
|
||
def build_fla_metadata(batch: "Batch", device: torch.device, pool: LinearStatePool) -> FLAMetadata:
|
||
"""Build the per-forward GDN metadata. Uses pinned host staging + non_blocking H2D
|
||
(the input_ids/attn-metadata pattern), so the copies overlap the forward instead of
|
||
stalling it.
|
||
|
||
Decode and verify run ``rows_per_req`` rows per request, so ``cu_seqlens`` is
|
||
``arange(bs+1) * rows_per_req``. Under CUDA graph ``LinearAttnBackend`` copies this
|
||
metadata into its persistent buffers (stable addresses) before each replay.
|
||
"""
|
||
reqs = batch.padded_reqs
|
||
pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()}
|
||
|
||
if not batch.is_prefill:
|
||
# every request continues its live state
|
||
bs = len(reqs)
|
||
idx_host = torch.tensor([gdn_slot(r) for r in reqs], dtype=torch.int32, **pin)
|
||
metadata = FLAMetadata(
|
||
cu_seqlens=torch.arange(0, (bs + 1) * batch.rows_per_req, batch.rows_per_req, dtype=torch.int32, device=device),
|
||
cache_indices=idx_host.to(device, non_blocking=True),
|
||
)
|
||
if batch.phase == "verify":
|
||
metadata.verify_scratch = pool.verify_scratch
|
||
return metadata
|
||
|
||
# prefill: cumsum of query (extend) lengths, per-request slot + continuation flags.
|
||
lens = [r.extend_len for r in reqs]
|
||
cu_host = torch.tensor([0, *lens], dtype=torch.int64, **pin).cumsum_(0)
|
||
idx_host = torch.tensor([gdn_slot(r) for r in reqs], dtype=torch.int32, **pin)
|
||
has_init_host = torch.tensor([r.cached_len > 0 for r in reqs], dtype=torch.bool, **pin)
|
||
fresh = [gdn_slot(r) for r in reqs if r.cached_len == 0]
|
||
fresh_host = torch.tensor(fresh, dtype=torch.int64, **pin) if fresh else None
|
||
|
||
track = _build_track_metadata(reqs, cu_host, device, pin, pool.conv_states.shape[-1])
|
||
|
||
return FLAMetadata(
|
||
cu_seqlens=cu_host.to(device, non_blocking=True),
|
||
cache_indices=idx_host.to(device, non_blocking=True),
|
||
has_initial_state=has_init_host.to(device, non_blocking=True),
|
||
fresh_state_indices=(
|
||
fresh_host.to(device, non_blocking=True) if fresh_host is not None else None
|
||
),
|
||
**track,
|
||
)
|
||
|
||
|
||
def _build_track_metadata(reqs, cu_host, device, pin, km1: int):
|
||
"""Hybrid-radix (extra_buffer): for each request that crosses a ×CHUNK boundary this
|
||
prefill forward, snapshot its GDN state at the deepest mid-chunk boundary into its current
|
||
ping-pong slot. Returns the ``FLAMetadata`` track kwargs, all None when no request
|
||
tracks (non-hybrid, or all extends < CHUNK+1)."""
|
||
empty = dict(track_dst=None, track_h_row=None, track_conv_src=None, track_boundary_row=None)
|
||
if not any(r.mamba_ping_pong is not None for r in reqs):
|
||
return empty
|
||
from freetoken.kernel.fla.chunk import CHUNK_SIZE
|
||
from freetoken.kernel.fla.index import prepare_chunk_offsets
|
||
|
||
assert km1 <= CHUNK_SIZE, (
|
||
f"conv history {km1} exceeds CHUNK_SIZE {CHUNK_SIZE}: the snapshot window "
|
||
"would reach before this forward's first token"
|
||
)
|
||
boh = prepare_chunk_offsets(cu_host, CHUNK_SIZE).tolist()
|
||
dst, h_row, conv_src, boundary_rows = [], [], [], []
|
||
for i, r in enumerate(reqs):
|
||
if r.mamba_ping_pong is None:
|
||
continue
|
||
# deepest mid-chunk boundary strictly inside the extend (h has the per-chunk state;
|
||
# the exact extend-end / aligned-final state lives in the live slot -> finish-donate).
|
||
c = (r.extend_len - 1) // CHUNK_SIZE
|
||
if c < 1:
|
||
continue
|
||
off = int(cu_host[i])
|
||
boundary = r.cached_len + c * CHUNK_SIZE
|
||
dst.append(r.mamba_ping_pong[r.mamba_next_track_idx])
|
||
h_row.append(boh[i] + c)
|
||
conv_src.append([off + c * CHUNK_SIZE - km1 + j for j in range(km1)])
|
||
boundary_rows.append(off + c * CHUNK_SIZE)
|
||
r.mamba_last_track_seqlen = boundary
|
||
r.mamba_next_track_idx = 1 - r.mamba_next_track_idx
|
||
if not dst:
|
||
return empty
|
||
to = lambda xs, **kw: torch.tensor(xs, **pin, **kw).to(device, non_blocking=True)
|
||
return dict(
|
||
track_dst=to(dst, dtype=torch.int64),
|
||
track_h_row=to(h_row, dtype=torch.int64),
|
||
track_conv_src=to(conv_src, dtype=torch.int64),
|
||
track_boundary_row=to(boundary_rows, dtype=torch.int64),
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class FLACaptureData:
|
||
"""The static buffers the linear-attention layers' CUDA graphs read, sized for the largest captured batch."""
|
||
|
||
cu_seqlens: torch.Tensor
|
||
cache_indices: torch.Tensor
|
||
|
||
@classmethod
|
||
def create(cls, max_bs: int, device: torch.device, rows_per_req: int = 1) -> FLACaptureData:
|
||
return cls(
|
||
# constant under a graph: every request has exactly `rows_per_req` query rows
|
||
cu_seqlens=torch.arange(max_bs + 1, dtype=torch.int32, device=device) * rows_per_req,
|
||
cache_indices=torch.zeros(max_bs, dtype=torch.int32, device=device),
|
||
)
|
||
|
||
|
||
class LinearAttnBackend(BaseAttnBackend):
|
||
"""Metadata of the linear-attention layers (GatedDeltaNet, KDA), next to the attention backend: ``FLAMetadata`` per forward, ``FLACaptureData`` for CUDA graphs.
|
||
|
||
The layers run their own kernels and read ``batch.fla_metadata``, so ``forward`` is never called.
|
||
"""
|
||
|
||
def __init__(self, device: torch.device, pool: LinearStatePool) -> None:
|
||
self.device = device
|
||
self.pool = pool
|
||
self.capture: FLACaptureData | None = None
|
||
self.capture_bs: List[int] = []
|
||
|
||
def forward(
|
||
self,
|
||
q: torch.Tensor,
|
||
k: torch.Tensor,
|
||
v: torch.Tensor,
|
||
layer_id: int,
|
||
batch: "Batch",
|
||
attn_spec: AttentionSpec | None = None,
|
||
) -> torch.Tensor:
|
||
# the GDN and KDA layers call their fla kernels themselves on batch.fla_metadata
|
||
raise NotImplementedError("linear-attention layers run their own kernels")
|
||
|
||
def prepare_metadata(self, batch: "Batch") -> None:
|
||
batch.fla_metadata = build_fla_metadata(batch, self.device, self.pool)
|
||
|
||
def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:
|
||
assert self.capture is None, "Capture already initialized."
|
||
self.capture = FLACaptureData.create(max(bs_list), self.device, rows_per_req=rows_per_req)
|
||
self.capture_bs = sorted(bs_list)
|
||
|
||
def prepare_for_capture(self, batch: "Batch") -> None:
|
||
assert (bs := batch.padded_size) in self.capture_bs and self.capture
|
||
capture = self.capture
|
||
# capture on the padding request's state slot, so the captured gather/scatter touches scratch; replay writes the real slots
|
||
capture.cache_indices[:bs].fill_(gdn_slot(batch.padded_reqs[0]))
|
||
batch.fla_metadata = FLAMetadata(cu_seqlens=capture.cu_seqlens[: bs + 1], cache_indices=capture.cache_indices[:bs])
|
||
if batch.phase == "verify":
|
||
batch.fla_metadata.verify_scratch = self.pool.verify_scratch
|
||
|
||
def prepare_for_replay(self, batch: "Batch") -> None:
|
||
metadata, bs = batch.fla_metadata, batch.padded_size
|
||
assert self.capture is not None and bs in self.capture_bs
|
||
# cu_seqlens is the constant arange * rows_per_req under a graph (i.e. no-op)
|
||
self.capture.cache_indices[:bs].copy_(metadata.cache_indices)
|
||
|
||
|
||
__all__ = ["FLACaptureData", "FLAMetadata", "LinearAttnBackend", "build_fla_metadata"]
|