mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
refactor(attention): share the MLA leaves' forward-metadata entry (#3605)
## Summary
Each of the four paged MLA leaves carried its own copy of
`init_forward_metadata` and `override_num_extends`:
- `MLAAttnBackend` (`paged/mla.py`)
- `FlashMLABackend` (`paged/flashmla.py`)
- `CuteDSLMLABackend` (`paged/tokenspeed_mla.py`)
- `TRTLLMMLABackend` (`paged/trtllm_mla.py`)
Every entry took the same four steps:
1. reject a query shard;
2. refuse a decode forward;
3. build prefill metadata for the leading extend requests;
4. refresh decode metadata for a target's mixed or idle batch.
The four overrides were near-identical. Both methods now live in a new
`PagedMLABackend` in `paged/base.py`, and each leaf keeps only its
`_init_prefill_metadata`. `Dots3NoteSWABackend`, `AscendMLABackend` and
the DSA wrapper reach the shared entry through the leaves they extend or
wrap.
Where the copies differed:
- **FlashMLA** is the only leaf that reads `extend_with_prefix`, which
picks ragged or paged prefill. It records the flag in a small
`init_forward_metadata` wrapper before calling the shared entry. This
keeps the hook's arguments the same for every leaf, including the
subclasses and the tests that call it directly.
- **TRT-LLM MLA**'s hook now receives `page_table_cpu` and discards it,
as its entry did before.
- **Error messages** now name the leaf class:
- the decode-forward error reads `TRTLLMMLABackend decode metadata goes
through ...`; it used to say `trtllm_mla`;
- the query-shard error from `Dots3NoteSWABackend` and
`AscendMLABackend` names the subclass instead of `MLAAttnBackend`.
No kernel code changes.
## Test Plan
On GB200, main (5b26453b0) against this branch, with the engine and
scheduler built at that commit.
- [x] **Same metadata.** Each leaf, as target and as draft, ran the same
set of calls:
- extend without a prefix, with a prefix, and with a prefix long enough
for several chunk loops;
- mixed batches with one and with two decode requests;
- idle;
- the decode-mode and query-shard errors;
- `override_num_extends`, including an exception raised inside it.
After each call, every tensor in the prefill, chunked-prefill and decode
metadata was hashed, along with every tensor attribute of the leaf. That
gave 1238 hashes over the four leaves and the dots3 window leaf, and all
match main. The only differences are the two query-shard messages of the
dots3 window leaf, described above. Two runs of main give identical
hashes.
- [x] **Same host time.** Main's entry was rebuilt from its source as a
subclass of each leaf. Both versions ran in one process and alternated
call by call, 1000 calls each, on an extend with prefixes and on a mixed
batch. Every median is within 0.4% of main (0.998x to 1.004x).
- [x] **Same test results.** 35 runtime test files cover these leaves:
router and cache setup, MLA, FlashMLA, CuTe DSL and TRT-LLM MLA
metadata, block decode, tree verify, DSA selection, dots3, Kimi K3,
DFlash and the drafters. Both sides give 983 passed and 18 skipped, with
no failures.
- [x] **Every module imports.** All modules under
`tokenspeed.runtime.layers` and `tokenspeed.runtime.models` import on
both sides: 149 of 149.
- [x] **New test.** `test/runtime/test_mla_forward_metadata_entry.py`
builds real MLA, FlashMLA and TRT-LLM MLA leaves (12 cases) and checks
that:
- a target's mixed batch refreshes its decode requests;
- a draft's mixed batch leaves decode metadata to its wrapper;
- `override_num_extends` restores the count after an exception;
- a decode forward is refused.
It passes on main too.
- [x] pre-commit on the changed files.
Signed-off-by: Nicolas <nperrin.ai@gmail.com>
This commit is contained in:
@@ -41,7 +41,10 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from tokenspeed.runtime.layers.attention.backends.base import CachePoolBinding
|
||||
from tokenspeed.runtime.layers.attention.backends.base import (
|
||||
CachePoolBinding,
|
||||
reject_query_shard,
|
||||
)
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.tree_verify import (
|
||||
TreeCascadeRows,
|
||||
)
|
||||
@@ -597,3 +600,71 @@ class PagedAttentionBackend(CachePoolBinding, ABC):
|
||||
pool.get_mla_kv_buffer(layer, slots, dst_dtype)
|
||||
for slots in chunk_kv_indices_list
|
||||
)
|
||||
|
||||
|
||||
class PagedMLABackend(PagedAttentionBackend):
|
||||
"""An MLA leaf: extend requests lead, decode requests follow them."""
|
||||
|
||||
def init_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
num_extends: int,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_with_prefix: bool,
|
||||
query_shard: QueryShardPlan | None,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
reject_query_shard(query_shard, type(self).__name__)
|
||||
if not (forward_mode.is_extend_or_mixed() or forward_mode.is_idle()):
|
||||
raise RuntimeError(
|
||||
f"{type(self).__name__} decode metadata goes through "
|
||||
"refresh_decode_metadata; init_forward_metadata only serves "
|
||||
f"extend/mixed ({forward_mode})"
|
||||
)
|
||||
if forward_mode.is_extend_or_mixed():
|
||||
self._init_prefill_metadata(
|
||||
seq_lens=seq_lens[:num_extends],
|
||||
extend_prefix_lens=extend_prefix_lens[:num_extends],
|
||||
extend_prefix_lens_cpu=extend_prefix_lens_cpu[:num_extends],
|
||||
extend_seq_lens=extend_seq_lens[:num_extends],
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu[:num_extends],
|
||||
page_table=page_table[:num_extends],
|
||||
page_table_cpu=page_table_cpu,
|
||||
)
|
||||
# A target's mixed/idle decode requests refresh here; a draft's in its wrapper.
|
||||
if forward_mode.is_idle() or (forward_mode.is_mixed() and not self.is_draft):
|
||||
self.refresh_decode_metadata(
|
||||
bs, bs, seq_lens, page_table, num_extends=num_extends
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def _init_prefill_metadata(
|
||||
self,
|
||||
*,
|
||||
seq_lens: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Build the prefill metadata of the leading extend requests."""
|
||||
|
||||
@contextmanager
|
||||
def override_num_extends(self, num_extends: int):
|
||||
assert self.forward_decode_metadata is not None
|
||||
prev = self.forward_decode_metadata.num_extends
|
||||
self.forward_decode_metadata.num_extends = num_extends
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.forward_decode_metadata.num_extends = prev
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -38,10 +37,8 @@ from tokenspeed_kernel.ops.attention.mla.cuda import (
|
||||
|
||||
from tokenspeed.runtime.configs.model_config import AttentionArch
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
from tokenspeed.runtime.execution.query_shard import QueryShardPlan
|
||||
from tokenspeed.runtime.layers.attention.backends.base import reject_query_shard
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.base import (
|
||||
PagedAttentionBackend,
|
||||
PagedMLABackend,
|
||||
)
|
||||
from tokenspeed.runtime.layers.attention.chunk import (
|
||||
build_chunked_prefill_metadata_arrays,
|
||||
@@ -122,7 +119,7 @@ def _per_token_kv_indices(
|
||||
_global_workspace_buffer = None
|
||||
|
||||
|
||||
class FlashMLABackend(PagedAttentionBackend):
|
||||
class FlashMLABackend(PagedMLABackend):
|
||||
"""FlashMLA leaf for TokenSpeed scheduling.
|
||||
|
||||
Uses the FlashMLA kernel for decode (any q_len) and FlashInfer's MLA
|
||||
@@ -228,6 +225,8 @@ class FlashMLABackend(PagedAttentionBackend):
|
||||
# graph recorded alive for the graph's lifetime.
|
||||
self._decode_tile_metadata: object | None = None
|
||||
self._decode_tile_metadata_keepalive: list[object] = []
|
||||
# The current forward's extend_with_prefix, read by the prefill hook.
|
||||
self._extend_with_prefix = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Metadata init
|
||||
@@ -403,58 +402,11 @@ class FlashMLABackend(PagedAttentionBackend):
|
||||
self.prefill_wrapper_ragged
|
||||
)
|
||||
|
||||
def init_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
num_extends: int,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_with_prefix: bool,
|
||||
query_shard: QueryShardPlan | None,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
**kwargs,
|
||||
):
|
||||
reject_query_shard(query_shard, "FlashMLABackend")
|
||||
if not (forward_mode.is_extend_or_mixed() or forward_mode.is_idle()):
|
||||
raise RuntimeError(
|
||||
"FlashMLA decode metadata goes through refresh_decode_metadata; "
|
||||
f"init_forward_metadata only serves extend/mixed ({forward_mode})"
|
||||
)
|
||||
if forward_mode.is_extend_or_mixed():
|
||||
self._init_prefill_metadata(
|
||||
seq_lens=seq_lens[:num_extends],
|
||||
extend_with_prefix=extend_with_prefix,
|
||||
extend_prefix_lens=extend_prefix_lens[:num_extends],
|
||||
extend_prefix_lens_cpu=extend_prefix_lens_cpu[:num_extends],
|
||||
extend_seq_lens=extend_seq_lens[:num_extends],
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu[:num_extends],
|
||||
page_table=page_table[:num_extends],
|
||||
page_table_cpu=page_table_cpu,
|
||||
)
|
||||
# Target mixed/idle batches carry decode rows whose metadata this
|
||||
# init must cover; the same in-place refresh serves them. A draft's
|
||||
# decode metadata instead comes from the wrapper's refresh after this
|
||||
# init (the unified draft contract).
|
||||
if forward_mode.is_idle() or (forward_mode.is_mixed() and not self.is_draft):
|
||||
self.refresh_decode_metadata(
|
||||
bs, bs, seq_lens, page_table, num_extends=num_extends
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def override_num_extends(self, num_extends: int):
|
||||
assert self.forward_decode_metadata is not None
|
||||
prev = self.forward_decode_metadata.num_extends
|
||||
self.forward_decode_metadata.num_extends = num_extends
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.forward_decode_metadata.num_extends = prev
|
||||
def init_forward_metadata(self, *args, extend_with_prefix: bool, **kwargs) -> None:
|
||||
self._extend_with_prefix = extend_with_prefix
|
||||
super().init_forward_metadata(
|
||||
*args, extend_with_prefix=extend_with_prefix, **kwargs
|
||||
)
|
||||
|
||||
def _renew_decode_tile_metadata(self, *, for_graph: bool) -> None:
|
||||
"""Bind a fresh (uninitialized) FlashMLASchedMeta for the seq_lens just
|
||||
@@ -485,7 +437,6 @@ class FlashMLABackend(PagedAttentionBackend):
|
||||
def _init_prefill_metadata(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
extend_with_prefix: bool,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
@@ -495,7 +446,7 @@ class FlashMLABackend(PagedAttentionBackend):
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
):
|
||||
# EXTEND path — flashinfer ragged/paged prefill.
|
||||
extend_no_prefix = not extend_with_prefix
|
||||
extend_no_prefix = not self._extend_with_prefix
|
||||
if extend_no_prefix and bool(extend_prefix_lens_cpu.any()):
|
||||
# The ragged plan sizes its paged kv_indices by the prefix sum
|
||||
# (zero here), so a prefixed row would overrun it in-kernel.
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -44,11 +43,8 @@ from tokenspeed_kernel.signature import dense_tensor_format, format_signature
|
||||
|
||||
from tokenspeed.runtime.configs.model_config import AttentionArch
|
||||
from tokenspeed.runtime.configs.numerics import BITWISE_ENVELOPES
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
from tokenspeed.runtime.execution.query_shard import QueryShardPlan
|
||||
from tokenspeed.runtime.layers.attention.backends.base import reject_query_shard
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.base import (
|
||||
PagedAttentionBackend,
|
||||
PagedMLABackend,
|
||||
)
|
||||
from tokenspeed.runtime.layers.attention.backends.support import CudaGraphSupport
|
||||
from tokenspeed.runtime.layers.attention.chunk import (
|
||||
@@ -138,7 +134,7 @@ class MLADecodeMetadata:
|
||||
return self.seq_lens
|
||||
|
||||
|
||||
class MLAAttnBackend(PagedAttentionBackend):
|
||||
class MLAAttnBackend(PagedMLABackend):
|
||||
"""Unified MLA leaf routed through tokenspeed_kernel MLA APIs."""
|
||||
|
||||
supports_mla_projected_value_decode = True
|
||||
@@ -508,60 +504,6 @@ class MLAAttnBackend(PagedAttentionBackend):
|
||||
# Metadata
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def init_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
num_extends: int,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_with_prefix: bool,
|
||||
query_shard: QueryShardPlan | None,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
reject_query_shard(query_shard, "MLAAttnBackend")
|
||||
if not (forward_mode.is_extend_or_mixed() or forward_mode.is_idle()):
|
||||
raise RuntimeError(
|
||||
"MLA decode metadata goes through refresh_decode_metadata; "
|
||||
f"init_forward_metadata only serves extend/mixed ({forward_mode})"
|
||||
)
|
||||
if forward_mode.is_extend_or_mixed():
|
||||
self._init_prefill_metadata(
|
||||
seq_lens=seq_lens[:num_extends],
|
||||
extend_prefix_lens=extend_prefix_lens[:num_extends],
|
||||
extend_prefix_lens_cpu=extend_prefix_lens_cpu[:num_extends],
|
||||
extend_seq_lens=extend_seq_lens[:num_extends],
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu[:num_extends],
|
||||
page_table=page_table[:num_extends],
|
||||
page_table_cpu=page_table_cpu,
|
||||
)
|
||||
|
||||
# Target mixed/idle batches carry decode requests whose metadata this
|
||||
# init must cover (verify decodes q_len tokens per request); the same
|
||||
# in-place refresh serves them. A draft's decode metadata instead
|
||||
# comes from the wrapper's refresh after this init (the unified draft
|
||||
# contract).
|
||||
if forward_mode.is_idle() or (forward_mode.is_mixed() and not self.is_draft):
|
||||
self.refresh_decode_metadata(
|
||||
bs, bs, seq_lens, page_table, num_extends=num_extends
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def override_num_extends(self, num_extends: int):
|
||||
assert self.forward_decode_metadata is not None
|
||||
prev = self.forward_decode_metadata.num_extends
|
||||
self.forward_decode_metadata.num_extends = num_extends
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.forward_decode_metadata.num_extends = prev
|
||||
|
||||
def _init_prefill_metadata(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
|
||||
@@ -30,7 +30,6 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -47,12 +46,9 @@ from tokenspeed_kernel.ops.attention.mla.tokenspeed_mla import (
|
||||
from tokenspeed_kernel.ops.metadata import broadcast_block_lengths
|
||||
|
||||
from tokenspeed.runtime.configs.model_config import AttentionArch
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
from tokenspeed.runtime.execution.query_shard import QueryShardPlan
|
||||
from tokenspeed.runtime.execution.workspace import workspace_pool
|
||||
from tokenspeed.runtime.layers.attention.backends.base import reject_query_shard
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.base import (
|
||||
PagedAttentionBackend,
|
||||
PagedMLABackend,
|
||||
)
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.tree_verify import (
|
||||
TreeCascadeRows,
|
||||
@@ -192,7 +188,7 @@ class _DCPDecodeState:
|
||||
return visible
|
||||
|
||||
|
||||
class CuteDSLMLABackend(PagedAttentionBackend):
|
||||
class CuteDSLMLABackend(PagedMLABackend):
|
||||
"""CuteDSL MLA leaf for Blackwell SM100 GPUs.
|
||||
|
||||
Decode uses CuTe DSL JIT-compiled kernels via tokenspeed_mla_decode().
|
||||
@@ -369,59 +365,6 @@ class CuteDSLMLABackend(PagedAttentionBackend):
|
||||
|
||||
# ---- Metadata initialization ----
|
||||
|
||||
def init_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
num_extends: int,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_with_prefix: bool,
|
||||
query_shard: QueryShardPlan | None,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
**kwargs,
|
||||
):
|
||||
reject_query_shard(query_shard, "CuteDSLMLABackend")
|
||||
if not (forward_mode.is_extend_or_mixed() or forward_mode.is_idle()):
|
||||
raise RuntimeError(
|
||||
"tokenspeed_mla decode metadata goes through "
|
||||
"refresh_decode_metadata; init_forward_metadata only serves "
|
||||
f"extend/mixed ({forward_mode})"
|
||||
)
|
||||
if forward_mode.is_extend_or_mixed():
|
||||
self._init_prefill_metadata(
|
||||
seq_lens[:num_extends],
|
||||
page_table=page_table[:num_extends],
|
||||
page_table_cpu=page_table_cpu,
|
||||
extend_prefix_lens=extend_prefix_lens[:num_extends],
|
||||
extend_prefix_lens_cpu=extend_prefix_lens_cpu[:num_extends],
|
||||
extend_seq_lens=extend_seq_lens[:num_extends],
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu[:num_extends],
|
||||
)
|
||||
# Target mixed/idle batches carry decode requests whose metadata this
|
||||
# init must cover; the same in-place refresh serves them. A draft's
|
||||
# decode metadata instead comes from the wrapper's refresh after this
|
||||
# init (the unified draft contract).
|
||||
if forward_mode.is_idle() or (forward_mode.is_mixed() and not self.is_draft):
|
||||
self.refresh_decode_metadata(
|
||||
bs, bs, seq_lens, page_table, num_extends=num_extends
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def override_num_extends(self, num_extends: int):
|
||||
assert self.forward_decode_metadata is not None
|
||||
prev = self.forward_decode_metadata.num_extends
|
||||
self.forward_decode_metadata.num_extends = num_extends
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.forward_decode_metadata.num_extends = prev
|
||||
|
||||
def _init_prefill_metadata(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
|
||||
@@ -24,7 +24,6 @@ MLA attention leaf using fused kernels optimized for SM100 (Blackwell) GPUs.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -36,11 +35,8 @@ from tokenspeed_kernel.ops.attention.mha.flashinfer import (
|
||||
from tokenspeed_kernel.ops.attention.tree import tree_window_attention
|
||||
|
||||
from tokenspeed.runtime.configs.model_config import AttentionArch
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
from tokenspeed.runtime.execution.query_shard import QueryShardPlan
|
||||
from tokenspeed.runtime.layers.attention.backends.base import reject_query_shard
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.base import (
|
||||
PagedAttentionBackend,
|
||||
PagedMLABackend,
|
||||
)
|
||||
from tokenspeed.runtime.layers.attention.backends.paged.tree_verify import (
|
||||
TreeCascadeRows,
|
||||
@@ -196,7 +192,7 @@ class TRTLLMMLADecodeMetadata:
|
||||
_dsa_kv_seq_lens: torch.Tensor | None = None
|
||||
|
||||
|
||||
class TRTLLMMLABackend(PagedAttentionBackend):
|
||||
class TRTLLMMLABackend(PagedMLABackend):
|
||||
"""trtllm_mla leaf using fused kernels."""
|
||||
|
||||
default_kernel_page_size = TRTLLM_MLA_DEFAULT_PAGE_SIZE
|
||||
@@ -246,58 +242,6 @@ class TRTLLMMLABackend(PagedAttentionBackend):
|
||||
self.forward_prefill_metadata = None
|
||||
self.chunked_prefill_metadata = None
|
||||
|
||||
def init_forward_metadata(
|
||||
self,
|
||||
bs: int,
|
||||
num_extends: int,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_with_prefix: bool,
|
||||
query_shard: QueryShardPlan | None,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
**kwargs,
|
||||
):
|
||||
reject_query_shard(query_shard, "TRTLLMMLABackend")
|
||||
del page_table_cpu
|
||||
if not (forward_mode.is_extend_or_mixed() or forward_mode.is_idle()):
|
||||
raise RuntimeError(
|
||||
"trtllm_mla decode metadata goes through refresh_decode_metadata; "
|
||||
f"init_forward_metadata only serves extend/mixed ({forward_mode})"
|
||||
)
|
||||
if forward_mode.is_extend_or_mixed():
|
||||
self._init_prefill_metadata(
|
||||
seq_lens[:num_extends],
|
||||
page_table=page_table[:num_extends],
|
||||
extend_prefix_lens=extend_prefix_lens[:num_extends],
|
||||
extend_prefix_lens_cpu=extend_prefix_lens_cpu[:num_extends],
|
||||
extend_seq_lens=extend_seq_lens[:num_extends],
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu[:num_extends],
|
||||
)
|
||||
# Target mixed/idle batches carry decode rows whose metadata this
|
||||
# init must cover; the same in-place refresh serves them. A draft's
|
||||
# decode metadata instead comes from the wrapper's refresh after this
|
||||
# init (the unified draft contract).
|
||||
if forward_mode.is_idle() or (forward_mode.is_mixed() and not self.is_draft):
|
||||
self.refresh_decode_metadata(
|
||||
bs, bs, seq_lens, page_table, num_extends=num_extends
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def override_num_extends(self, num_extends: int):
|
||||
assert self.forward_decode_metadata is not None
|
||||
prev = self.forward_decode_metadata.num_extends
|
||||
self.forward_decode_metadata.num_extends = num_extends
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.forward_decode_metadata.num_extends = prev
|
||||
|
||||
def _init_prefill_metadata(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -306,7 +250,10 @@ class TRTLLMMLABackend(PagedAttentionBackend):
|
||||
extend_prefix_lens_cpu: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
extend_seq_lens_cpu: torch.Tensor,
|
||||
*,
|
||||
page_table_cpu: torch.Tensor | None,
|
||||
):
|
||||
del page_table_cpu
|
||||
max_seq_len = self.max_context_len
|
||||
cum_seq_lens = torch.zeros(
|
||||
len(seq_lens) + 1, dtype=torch.int32, device=seq_lens.device
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""The MLA leaves' shared extend/mixed metadata entry."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from ci_system.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=60,
|
||||
suite="runtime-1gpu",
|
||||
disabled_on_runners=["amd-*"],
|
||||
disabled_on_runners_reason="The FlashMLA and TRT-LLM MLA leaves are NVIDIA-only.",
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
LEAVES = (
|
||||
"mla",
|
||||
"flashmla",
|
||||
pytest.param(
|
||||
"trtllm_mla",
|
||||
marks=pytest.mark.skipif(
|
||||
torch.cuda.is_available() and torch.cuda.get_device_capability()[0] != 10,
|
||||
reason="TRT-LLM MLA kernels need SM100",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_leaf(monkeypatch):
|
||||
from test.runtime.conftest import make_kimi_pool
|
||||
|
||||
from tokenspeed.runtime.layers.attention.backends.paged import (
|
||||
flashmla,
|
||||
mla,
|
||||
trtllm_mla,
|
||||
)
|
||||
from tokenspeed.runtime.layers.attention.configs.base import AttnConfig
|
||||
from tokenspeed.runtime.layers.attention.configs.mla import MLAConfig
|
||||
from tokenspeed.runtime.utils.env import global_server_args_dict
|
||||
|
||||
monkeypatch.setitem(global_server_args_dict, "mla_disable_ragged", False)
|
||||
monkeypatch.setitem(global_server_args_dict, "chunked_prefill_size", 256)
|
||||
monkeypatch.setitem(global_server_args_dict, "mla_chunk_multiplier", 1)
|
||||
leaves = {
|
||||
"mla": mla.MLAAttnBackend,
|
||||
"flashmla": flashmla.FlashMLABackend,
|
||||
"trtllm_mla": trtllm_mla.TRTLLMMLABackend,
|
||||
}
|
||||
|
||||
def make(backend: str, *, is_draft: bool):
|
||||
spec = MLAConfig(
|
||||
backend_name=backend,
|
||||
num_attention_heads=16,
|
||||
num_kv_heads=1,
|
||||
head_dim=192,
|
||||
attn_tp_size=1,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
scaling=192**-0.5,
|
||||
kv_cache_dim=576,
|
||||
)
|
||||
config = AttnConfig(
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=torch.bfloat16,
|
||||
kernel_page_size=64,
|
||||
context_len=512,
|
||||
max_bs=4,
|
||||
max_resident_requests=4,
|
||||
kv_cache_quant_method="",
|
||||
is_draft=is_draft,
|
||||
components=(spec,),
|
||||
)
|
||||
leaf = leaves[backend](config, spec, kernel_page_size=64)
|
||||
leaf.set_cache_pool(make_kimi_pool("cuda", usable_pages=6))
|
||||
leaf.init_cuda_graph_state(max_bs=4)
|
||||
return leaf
|
||||
|
||||
return make
|
||||
|
||||
|
||||
def _init(leaf, mode) -> None:
|
||||
# Two extend requests with cached prefixes, then one decode request.
|
||||
prefix_cpu = torch.tensor([65, 17], dtype=torch.int32)
|
||||
extend_cpu = torch.tensor([4, 3], dtype=torch.int32)
|
||||
table_cpu = torch.tensor(
|
||||
[[2, 5, 1, 0, 0, 0, 0, 0], [4, 3, 1, 0, 0, 0, 0, 0], [1] * 8],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
leaf.init_forward_metadata(
|
||||
3,
|
||||
2,
|
||||
torch.tensor([69, 20, 8], dtype=torch.int32, device="cuda"),
|
||||
table_cpu.cuda(),
|
||||
mode,
|
||||
extend_seq_lens=extend_cpu.cuda(),
|
||||
extend_seq_lens_cpu=extend_cpu,
|
||||
extend_prefix_lens=prefix_cpu.cuda(),
|
||||
extend_prefix_lens_cpu=prefix_cpu,
|
||||
extend_with_prefix=True,
|
||||
query_shard=None,
|
||||
page_table_cpu=table_cpu[:2],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", LEAVES)
|
||||
def test_mixed_target_refreshes_its_decode_requests(make_leaf, backend):
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
|
||||
leaf = make_leaf(backend, is_draft=False)
|
||||
_init(leaf, ForwardMode.MIXED)
|
||||
assert leaf.chunked_prefill_metadata.extend_seq_lens_cpu.tolist() == [4, 3]
|
||||
assert leaf.forward_decode_metadata.num_extends == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", LEAVES)
|
||||
def test_mixed_draft_leaves_decode_metadata_to_its_wrapper(make_leaf, backend):
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
|
||||
leaf = make_leaf(backend, is_draft=True)
|
||||
_init(leaf, ForwardMode.MIXED)
|
||||
assert leaf.chunked_prefill_metadata.extend_seq_lens_cpu.tolist() == [4, 3]
|
||||
assert leaf.forward_decode_metadata is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", LEAVES)
|
||||
def test_override_num_extends_restores_after_an_error(make_leaf, backend):
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
|
||||
leaf = make_leaf(backend, is_draft=False)
|
||||
_init(leaf, ForwardMode.MIXED)
|
||||
with pytest.raises(KeyError):
|
||||
with leaf.override_num_extends(0):
|
||||
assert leaf.forward_decode_metadata.num_extends == 0
|
||||
raise KeyError("draft step failed")
|
||||
assert leaf.forward_decode_metadata.num_extends == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", LEAVES)
|
||||
def test_decode_forward_is_refused(make_leaf, backend):
|
||||
from tokenspeed.runtime.execution.forward_batch_info import ForwardMode
|
||||
|
||||
leaf = make_leaf(backend, is_draft=False)
|
||||
with pytest.raises(RuntimeError, match="init_forward_metadata only serves"):
|
||||
_init(leaf, ForwardMode.DECODE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user