mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor: add engine and scheduler seams for multi-row decode steps * refactor: rename the graph steps argument to rows_per_req * refactor: drop VerifyInfo and require rows_per_req on verify batches * refactor: fix stale seam comments and read decode_rows_per_req directly * refactor: reword two GraphRunner comments for multi-row graphs * fix(engine): scale the GPU-decode MoE pool limits by decode_rows_per_req * refactor: drop the unread Context.moe_offload_cache field * test: trim the multi-row seam tests
337 lines
14 KiB
Python
337 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
from functools import cached_property
|
|
from typing import TYPE_CHECKING, Dict, List, Literal
|
|
|
|
import torch
|
|
from freetoken.core import Batch, get_global_ctx
|
|
from freetoken.distributed import get_tp_info
|
|
from freetoken.env import ENV
|
|
from freetoken.utils import div_even, init_logger
|
|
|
|
from .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadata
|
|
from .utils import BaseCaptureData
|
|
|
|
if TYPE_CHECKING:
|
|
from flashinfer import (
|
|
BatchDecodeWithPagedKVCacheWrapper,
|
|
BatchPrefillWithPagedKVCacheWrapper,
|
|
CUDAGraphBatchDecodeWithPagedKVCacheWrapper,
|
|
)
|
|
from freetoken.models import ModelConfig
|
|
|
|
|
|
def _next_power_of_2(n: int) -> int:
|
|
if n <= 1:
|
|
return 1
|
|
return 1 << math.ceil(math.log2(n))
|
|
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class FICaptureData(BaseCaptureData):
|
|
@property
|
|
def one_tensor(self) -> torch.Tensor:
|
|
return self.seq_lens
|
|
|
|
@property
|
|
def indices(self) -> torch.Tensor:
|
|
return self.page_table
|
|
|
|
|
|
@dataclass
|
|
class FIMetadata(BaseAttnMetadata):
|
|
# fmt: off
|
|
cu_seqlens_q_cpu: torch.Tensor # on cpu
|
|
cu_seqlens_k_cpu: torch.Tensor # on cpu
|
|
cu_seqlens_q_gpu: torch.Tensor # on gpu
|
|
indices: torch.Tensor # on gpu
|
|
last_page_len_cpu: torch.Tensor # on cpu
|
|
num_qo_heads: int
|
|
num_kv_heads: int
|
|
head_dim: int
|
|
page_size: Literal[1] # currently only support page_size=1
|
|
pos_encoding_mode: str
|
|
seq_lens_cpu: torch.Tensor # on cpu
|
|
dtype: torch.dtype
|
|
wrapper: BatchPrefillWithPagedKVCacheWrapper | BatchDecodeWithPagedKVCacheWrapper
|
|
initialized: bool = False
|
|
# fmt: on
|
|
|
|
def __post_init__(self) -> None:
|
|
assert self.page_size == 1, "Currently only page_size=1 is supported."
|
|
assert (
|
|
self.cu_seqlens_k_cpu.is_cpu
|
|
and self.cu_seqlens_q_cpu.is_cpu
|
|
and self.cu_seqlens_q_gpu.is_cuda
|
|
and self.indices.is_cuda
|
|
and self.last_page_len_cpu.is_cpu
|
|
and self.seq_lens_cpu.is_cpu
|
|
)
|
|
|
|
def get_last_indices(self, bs: int) -> torch.Tensor:
|
|
return self.cu_seqlens_q_gpu[1 : 1 + bs] - 1
|
|
|
|
|
|
class FlashInferBackend(BaseAttnBackend):
|
|
def __init__(self, config: ModelConfig) -> None:
|
|
from flashinfer import (
|
|
BatchDecodeWithPagedKVCacheWrapper,
|
|
BatchPrefillWithPagedKVCacheWrapper,
|
|
)
|
|
|
|
self.config = config
|
|
self.kvcache = get_global_ctx().kv_cache
|
|
self.device = self.kvcache.device
|
|
# fa2 split-KV prefill needs ``tmp_v <= qo_heads_local * padded_batch_size *
|
|
# cta_tile_q * head_dim * 4`` bytes of scratch, where flashinfer's scheduler
|
|
# caps ``padded_batch_size`` at ~``2 * SM / kv_heads_local`` and
|
|
# ``cta_tile_q`` is 128 (64 at head_dim >= 256); when the cap can't be met
|
|
# it disables split-KV and allocates no tmp_v at all. The original flat
|
|
# 128 MiB overflowed on head_dim=256 extend-prefills (Qwen3.5/3.6 MoE) and
|
|
# the flat 256 MiB on MiniMax-M3's 64-head dense layers (H100, 132 SMs:
|
|
# 64 heads x ceil(2*132/4)=66 padded rows x 128 x 128 x 4 B = 264 MiB of
|
|
# tmp_v, over the flat buffer). Derive the bound
|
|
# from the model's TP-LOCAL geometry + this device's SM count, with slack
|
|
# for the tmp_s/merge siblings, floored at the old 256 MiB -- geometries
|
|
# that never exceeded the flat buffer (e.g. GLM-4.7's 96q/8kv) stay at it.
|
|
tp_size = get_tp_info().size
|
|
qo_local = div_even(config.num_qo_heads, tp_size)
|
|
kv_local = div_even(config.num_kv_heads, tp_size, allow_replicate=True)
|
|
sm_count = (
|
|
torch.cuda.get_device_properties(self.device).multi_processor_count
|
|
if self.device.type == "cuda"
|
|
else 128
|
|
)
|
|
cta_tile_q = 64 if config.head_dim >= 256 else 128
|
|
padded_batch = -(-2 * sm_count // max(1, kv_local))
|
|
tmp_v_bound = qo_local * padded_batch * cta_tile_q * config.head_dim * 4
|
|
workspace_bytes = max(256 * 1024 * 1024, tmp_v_bound + 32 * 1024 * 1024)
|
|
self.float_workspace_buffer = torch.empty(
|
|
workspace_bytes, dtype=torch.uint8, device=self.device
|
|
)
|
|
self.prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
|
|
self.float_workspace_buffer,
|
|
kv_layout="NHD",
|
|
backend="fa2", # flashinfer fa3 is slow, use fa2 instead
|
|
)
|
|
self.decode_wrappers = BatchDecodeWithPagedKVCacheWrapper(
|
|
self.float_workspace_buffer,
|
|
use_tensor_cores=self.use_tensor_cores,
|
|
kv_layout="NHD",
|
|
backend="fa2", # flashinfer fa3 is slow, use fa2 instead
|
|
)
|
|
|
|
# NOTE: some hack to reuse the int_workspace_buffer
|
|
self.int_workspace_buffer = self.prefill_wrapper._int_workspace_buffer
|
|
self.decode_wrappers._int_workspace_buffer = self.int_workspace_buffer
|
|
|
|
# initialize some data members
|
|
tp_size = get_tp_info().size
|
|
self.qo_head_local = div_even(self.config.num_qo_heads, tp_size)
|
|
self.kv_head_local = div_even(self.config.num_kv_heads, tp_size, allow_replicate=True)
|
|
|
|
self.cached_ones_cpu: torch.Tensor = torch.tensor([], dtype=torch.int32, pin_memory=True)
|
|
# for cuda graph
|
|
self.capture_bs: List[int] = []
|
|
self.max_graph_bs = 0
|
|
self.graph_wrappers: Dict[
|
|
int, CUDAGraphBatchDecodeWithPagedKVCacheWrapper | BatchPrefillWithPagedKVCacheWrapper
|
|
] = {}
|
|
self.capture: FICaptureData | None = None
|
|
self.capture_rows_per_req = 1
|
|
self.last_event = torch.cuda.Event()
|
|
self.last_event.record()
|
|
|
|
def _initialize_metadata_once(self, metadata: FIMetadata) -> None:
|
|
if metadata.initialized:
|
|
return
|
|
|
|
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
|
|
|
|
metadata.initialized = True
|
|
# FlashInfer planning reuses a pinned host staging buffer and launches an
|
|
# async H2D copy. Wait here before the next plan mutates that host buffer.
|
|
self.last_event.synchronize()
|
|
if isinstance(metadata.wrapper, BatchDecodeWithPagedKVCacheWrapper):
|
|
metadata.wrapper.plan(
|
|
indptr=metadata.cu_seqlens_k_cpu,
|
|
indices=metadata.indices,
|
|
last_page_len=metadata.last_page_len_cpu,
|
|
num_qo_heads=metadata.num_qo_heads,
|
|
num_kv_heads=metadata.num_kv_heads,
|
|
head_dim=metadata.head_dim,
|
|
page_size=metadata.page_size,
|
|
pos_encoding_mode=metadata.pos_encoding_mode,
|
|
seq_lens=metadata.seq_lens_cpu,
|
|
data_type=metadata.dtype,
|
|
q_data_type=metadata.dtype,
|
|
kv_data_type=metadata.dtype,
|
|
non_blocking=True,
|
|
)
|
|
else:
|
|
metadata.wrapper.plan(
|
|
qo_indptr=metadata.cu_seqlens_q_cpu,
|
|
paged_kv_indptr=metadata.cu_seqlens_k_cpu,
|
|
paged_kv_indices=metadata.indices,
|
|
paged_kv_last_page_len=metadata.last_page_len_cpu,
|
|
num_qo_heads=metadata.num_qo_heads,
|
|
num_kv_heads=metadata.num_kv_heads,
|
|
head_dim_qk=metadata.head_dim,
|
|
page_size=metadata.page_size,
|
|
pos_encoding_mode=metadata.pos_encoding_mode,
|
|
seq_lens=metadata.seq_lens_cpu,
|
|
q_data_type=metadata.dtype,
|
|
kv_data_type=metadata.dtype,
|
|
non_blocking=True,
|
|
causal=True,
|
|
)
|
|
self.last_event.record()
|
|
|
|
def _get_ones_cpu(self, bs: int) -> torch.Tensor:
|
|
if bs <= len(self.cached_ones_cpu):
|
|
return self.cached_ones_cpu[:bs]
|
|
# padding to next pow of 2
|
|
next_len = _next_power_of_2(bs)
|
|
self.cached_ones_cpu = torch.ones(next_len, dtype=torch.int32, pin_memory=True)
|
|
return self.cached_ones_cpu[:bs]
|
|
|
|
def forward(
|
|
self,
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
layer_id: int,
|
|
batch: Batch,
|
|
attn_spec: AttentionSpec | None = None,
|
|
) -> torch.Tensor:
|
|
if attn_spec is not None:
|
|
# This backend has no window/sinks/sm_scale plumbing; dropping the spec
|
|
# silently would attend with the wrong scale or an unbounded window.
|
|
raise ValueError("The fi backend does not support per-call AttentionSpec.")
|
|
|
|
def _flatten_cache(cache: torch.Tensor) -> torch.Tensor: # treat page = 1
|
|
return cache.view(-1, 1, cache.shape[2], cache.shape[3])
|
|
|
|
metadata = batch.attn_metadata
|
|
assert isinstance(metadata, FIMetadata)
|
|
self._initialize_metadata_once(metadata)
|
|
self.kvcache.store_kv(k, v, batch.out_loc, layer_id)
|
|
kv_cache = (self.kvcache.k_cache(layer_id), self.kvcache.v_cache(layer_id))
|
|
kv_cache = (_flatten_cache(kv_cache[0]), _flatten_cache(kv_cache[1]))
|
|
return metadata.wrapper.run(q=q, paged_kv_cache=kv_cache)
|
|
|
|
def prepare_metadata(self, batch: Batch) -> None:
|
|
reqs = batch.padded_reqs
|
|
|
|
padded_size = len(reqs)
|
|
seqlens_q = [req.extend_len for req in reqs]
|
|
seqlens_k = [req.device_len for req in reqs]
|
|
cached_lens = [req.cached_len for req in reqs]
|
|
max_seqlen_q = max(seqlens_q)
|
|
CPU_KWARGS = {"device": "cpu", "dtype": torch.int32, "pin_memory": True}
|
|
|
|
device = self.device
|
|
seq_len_cpu = torch.tensor(seqlens_k, **CPU_KWARGS)
|
|
cu_seqlens_k_cpu = torch.tensor([0] + seqlens_k, **CPU_KWARGS).cumsum_(dim=0)
|
|
if max_seqlen_q == 1: # decode with all extend_len = 1
|
|
cu_seqlens_q_cpu = torch.arange(0, padded_size + 1, **CPU_KWARGS)
|
|
elif all(l == 0 for l in cached_lens): # prefill with no cache hit
|
|
cu_seqlens_q_cpu = cu_seqlens_k_cpu
|
|
else: # normal extend prefill, with partial cache hit
|
|
cu_seqlens_q_cpu = torch.tensor([0] + seqlens_q, **CPU_KWARGS).cumsum_(dim=0)
|
|
|
|
page_table = get_global_ctx().page_table
|
|
batch.attn_metadata = FIMetadata(
|
|
cu_seqlens_q_cpu=cu_seqlens_q_cpu,
|
|
cu_seqlens_k_cpu=cu_seqlens_k_cpu,
|
|
cu_seqlens_q_gpu=cu_seqlens_q_cpu.to(device, non_blocking=True),
|
|
indices=torch.cat([page_table[req.table_idx, : req.device_len] for req in reqs]),
|
|
last_page_len_cpu=self._get_ones_cpu(padded_size),
|
|
num_qo_heads=self.qo_head_local,
|
|
num_kv_heads=self.kv_head_local,
|
|
head_dim=self.config.head_dim,
|
|
page_size=1,
|
|
pos_encoding_mode="NONE",
|
|
seq_lens_cpu=seq_len_cpu,
|
|
dtype=self.kvcache.dtype,
|
|
wrapper=self.decode_wrappers if batch.is_decode else self.prefill_wrapper,
|
|
)
|
|
|
|
def reset_capture(self) -> None:
|
|
# Base clears the common capture scratch; additionally drop the per-bs decode graph
|
|
# wrappers (their indptr/indices alias freed capture scratch). Preserves the
|
|
# long-lived workspace buffers. Lets init_capture_graph re-run after a cache rebuild.
|
|
super().reset_capture()
|
|
self.graph_wrappers = {}
|
|
|
|
def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:
|
|
assert self.capture is None, "Capture already initialized."
|
|
max_bs = max(bs_list)
|
|
capture = FICaptureData.create(max_bs, max_seq_len, self.kvcache.device, rows_per_req=rows_per_req)
|
|
capture.page_table = capture.page_table.view(-1) # use 1D as ragged indices
|
|
self.max_graph_bs = max_bs
|
|
self.capture = capture
|
|
self.capture_bs = sorted(bs_list)
|
|
self.capture_rows_per_req = rows_per_req
|
|
|
|
@cached_property
|
|
def use_tensor_cores(self) -> bool:
|
|
if (overriden_value := ENV.FLASHINFER_USE_TENSOR_CORES.value) is not None:
|
|
logger.warning(f"Overriding FlashInfer tensor core usage to {overriden_value}")
|
|
return overriden_value
|
|
GQA = self.config.num_qo_heads // self.config.num_kv_heads
|
|
return GQA >= 4
|
|
|
|
def _make_graph_wrapper(
|
|
self, bs: int
|
|
) -> CUDAGraphBatchDecodeWithPagedKVCacheWrapper | BatchPrefillWithPagedKVCacheWrapper:
|
|
from flashinfer import BatchPrefillWithPagedKVCacheWrapper, CUDAGraphBatchDecodeWithPagedKVCacheWrapper
|
|
|
|
capture = self.capture
|
|
assert capture is not None
|
|
if self.capture_rows_per_req > 1:
|
|
wrapper = BatchPrefillWithPagedKVCacheWrapper(
|
|
self.float_workspace_buffer,
|
|
kv_layout="NHD",
|
|
backend="fa2",
|
|
use_cuda_graph=True,
|
|
qo_indptr_buf=capture.cu_seqlens_q[: bs + 1],
|
|
paged_kv_indptr_buf=capture.cu_seqlens_k[: bs + 1],
|
|
paged_kv_indices_buf=capture.indices,
|
|
paged_kv_last_page_len_buf=capture.one_tensor[:bs],
|
|
)
|
|
else:
|
|
wrapper = CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
|
|
self.float_workspace_buffer,
|
|
kv_layout="NHD",
|
|
use_tensor_cores=self.use_tensor_cores,
|
|
indptr_buffer=capture.cu_seqlens_k[: bs + 1],
|
|
indices_buffer=capture.indices,
|
|
last_page_len_buffer=capture.one_tensor[:bs],
|
|
)
|
|
wrapper._backend = "fa2"
|
|
wrapper._int_workspace_buffer = self.int_workspace_buffer
|
|
return wrapper
|
|
|
|
def prepare_for_capture(self, batch: Batch) -> None:
|
|
bs = batch.size
|
|
assert bs in self.capture_bs and bs not in self.graph_wrappers and self.capture
|
|
self.graph_wrappers[bs] = self._make_graph_wrapper(bs)
|
|
self.prepare_metadata(batch)
|
|
metadata = batch.attn_metadata
|
|
assert isinstance(metadata, FIMetadata)
|
|
metadata.wrapper = self.graph_wrappers[bs]
|
|
self._initialize_metadata_once(metadata)
|
|
|
|
def prepare_for_replay(self, batch: Batch) -> None:
|
|
metadata, bs = batch.attn_metadata, batch.padded_size
|
|
assert isinstance(metadata, FIMetadata) and not metadata.initialized
|
|
assert self.capture is not None and bs in self.capture_bs
|
|
metadata.wrapper = self.graph_wrappers[bs]
|
|
self._initialize_metadata_once(metadata)
|