mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor: add engine and scheduler seams for multi-row decode steps * refactor: rename the graph steps argument to rows_per_req * refactor: drop VerifyInfo and require rows_per_req on verify batches * refactor: fix stale seam comments and read decode_rows_per_req directly * refactor: reword two GraphRunner comments for multi-row graphs * fix(engine): scale the GPU-decode MoE pool limits by decode_rows_per_req * refactor: drop the unread Context.moe_offload_cache field * test: trim the multi-row seam tests
132 lines
4.8 KiB
Python
132 lines
4.8 KiB
Python
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
from typing import TYPE_CHECKING, List
|
|
|
|
if TYPE_CHECKING:
|
|
import torch
|
|
from freetoken.core import Batch
|
|
|
|
|
|
class AttnType(str, Enum):
|
|
"""Attention-type taxonomy, one value per KV-pool family. A model's attention
|
|
groups map onto these (KVCacheGroupSpec.attn_type); each backend declares the
|
|
set it can serve (BackendInfo.supported_types)."""
|
|
|
|
FULL = "full" # uniform causal MHA/GQA -> MHAKVCache
|
|
SWA = "swa" # sliding-window hybrid (window/sinks) -> HybridSWAKVCache
|
|
MLA = "mla" # plain latent-KV MLA -> MLAKVCache
|
|
DSA = "dsa" # latent-KV MLA + DSA sparse indexer -> DSAKVCache
|
|
DSV4 = "dsv4" # DSV4 window+compressed sparse -> DSV4PagedKVCache
|
|
# DSV41 (DeepSeek-V4.1): window + cross-layer-shared compressed KV in packed fp8/fp4
|
|
# rows, Full/Reindex/Reuse layer modes -> DSV41PagedKVCache
|
|
DSV41 = "dsv41"
|
|
LINEAR = "linear" # GDN/mamba state layers -> LinearStatePool
|
|
# GQA block-sparse (MiniMax-M3): paged GQA K/V + a per-sparse-layer index-key
|
|
# slab; the indexer picks top-k 128-token blocks per query -> BSAKVCache
|
|
BSA = "bsa"
|
|
# QSA compressed-block sparse (Qwen3.8-Flash-Next): paged GQA K/V + a compressed
|
|
# index-key slab (one row per index_ratio tokens, row = slot // index_ratio) +
|
|
# a per-request pending ring for unclosed groups -> QSAKVCache
|
|
QSA = "qsa"
|
|
|
|
@property
|
|
def backend_driven(self) -> bool:
|
|
# LINEAR layers reach their kernels directly (fla ops + batch.fla_metadata),
|
|
# not through an attention backend, so they never constrain backend choice.
|
|
return self is not AttnType.LINEAR
|
|
|
|
|
|
@dataclass
|
|
class AttentionSpec:
|
|
sliding_window: int | None = None
|
|
sm_scale: float | None = None
|
|
sinks: torch.Tensor | None = None
|
|
# rows of a multimodal span (batch.mm_block_ends) also attend to the span's later keys
|
|
bidirectional_mm_blocks: bool = False
|
|
|
|
|
|
@dataclass
|
|
class BaseAttnMetadata(ABC):
|
|
@abstractmethod
|
|
def get_last_indices(self, bs: int) -> torch.Tensor: ...
|
|
|
|
|
|
class BaseAttnBackend(ABC):
|
|
@abstractmethod
|
|
def forward(
|
|
self,
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
layer_id: int,
|
|
batch: Batch,
|
|
attn_spec: AttentionSpec | None = None,
|
|
) -> torch.Tensor: ...
|
|
|
|
@abstractmethod
|
|
def prepare_metadata(self, batch: Batch) -> None: ...
|
|
|
|
@abstractmethod
|
|
def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:
|
|
"""Static buffers for graphs of ``rows_per_req`` query rows per request: 1 for decode, more for a uniform extend."""
|
|
|
|
@abstractmethod
|
|
def prepare_for_capture(self, batch: Batch) -> None: ...
|
|
|
|
@abstractmethod
|
|
def prepare_for_replay(self, batch: Batch) -> None: ...
|
|
|
|
def reset_capture(self) -> None:
|
|
"""Drop CUDA-graph capture scratch so ``init_capture_graph`` can re-run after a
|
|
runtime cache rebuild. The default clears the common capture state (guarded by
|
|
``hasattr`` so backends that hold only a subset, e.g. dsv4, are safe). Backends
|
|
with extra per-bs graph state (FlashInfer ``graph_wrappers``) override this."""
|
|
if hasattr(self, "capture"):
|
|
self.capture = None
|
|
if hasattr(self, "capture_bs"):
|
|
self.capture_bs = []
|
|
if hasattr(self, "max_graph_bs"):
|
|
self.max_graph_bs = 0
|
|
|
|
|
|
class HybridBackend(BaseAttnBackend):
|
|
def __init__(
|
|
self,
|
|
prefill_backend: BaseAttnBackend,
|
|
decode_backend: BaseAttnBackend,
|
|
) -> None:
|
|
self.prefill_backend = prefill_backend
|
|
self.decode_backend = decode_backend
|
|
|
|
def forward(
|
|
self,
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
layer_id: int,
|
|
batch: Batch,
|
|
attn_spec: AttentionSpec | None = None,
|
|
) -> torch.Tensor:
|
|
backend = self.prefill_backend if batch.is_prefill else self.decode_backend
|
|
return backend.forward(q, k, v, layer_id, batch, attn_spec=attn_spec)
|
|
|
|
def prepare_metadata(self, batch: Batch) -> None:
|
|
backend = self.prefill_backend if batch.is_prefill else self.decode_backend
|
|
return backend.prepare_metadata(batch)
|
|
|
|
def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:
|
|
self.decode_backend.init_capture_graph(max_seq_len, bs_list, rows_per_req)
|
|
|
|
def prepare_for_capture(self, batch: Batch) -> None:
|
|
self.decode_backend.prepare_for_capture(batch)
|
|
|
|
def prepare_for_replay(self, batch: Batch) -> None:
|
|
self.decode_backend.prepare_for_replay(batch)
|
|
|
|
def reset_capture(self) -> None:
|
|
# Only the decode backend is ever captured (see init_capture_graph above).
|
|
self.decode_backend.reset_capture()
|