mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
feat(kda): FlashInfer persistent KDA prefill backend
Add --kda-backend flashinfer: KDA prefill runs FlashInfer's persistent CuTe DSL scan on SM100/SM103 (head dim 128, heads divisible by eight, gate bound in [-5, 0)). It takes the BF16 gate and beta logits as produced and the V-major FP32 state the prefill gather yields, and advances that state in place. auto keeps its current resolution. FlashInfer's public recurrent_kda plans this kernel from the device boundaries with a tolist() on every call, a synchronizing D2H per KDA layer. The adapter plans from cu_seqlens_cpu with the kernel's planner, reuses the plan across layers that share boundaries, and calls its forward entry, repeating the public path's input checks. At the Kimi-K3 prefill shape (16K tokens, 96 heads, 1 to 16 sequences) a KDA scan takes 491-733 us of kernel time, against 1224-1426 us for cutedsl_kda including its gate and state casts. Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4edb65eae9
commit
8fcc298526
@@ -69,7 +69,7 @@ if TYPE_CHECKING:
|
||||
from tokenspeed.runtime.layers.attention.kv_cache.base import CachePool
|
||||
|
||||
|
||||
KDA_PREFILL_BACKENDS = ("auto", "fla", "flashkda", "cutedsl_kda")
|
||||
KDA_PREFILL_BACKENDS = ("auto", "fla", "flashkda", "cutedsl_kda", "flashinfer")
|
||||
|
||||
|
||||
def resolve_kda_backend(kda_backend: str) -> str:
|
||||
@@ -78,8 +78,8 @@ def resolve_kda_backend(kda_backend: str) -> str:
|
||||
On AMD, the backend policy is ignored and compatible kernels are selected
|
||||
using registry priority. On NVIDIA, ``auto`` picks ``cutedsl_kda`` when its
|
||||
device-specific implementation is available, ``flashkda`` on SM90+, and
|
||||
``fla`` otherwise. Explicit CuteDSL selection is validated against device
|
||||
support. Decode is unaffected.
|
||||
``fla`` otherwise. Explicit CuteDSL and FlashInfer selections are validated
|
||||
against device support. Decode is unaffected.
|
||||
|
||||
Args:
|
||||
kda_backend: Normalized prefill backend name from the launch arguments.
|
||||
@@ -107,6 +107,15 @@ def resolve_kda_backend(kda_backend: str) -> str:
|
||||
raise ValueError(
|
||||
"--kda-backend cutedsl_kda requires an NVIDIA sm_100 or sm_103 device"
|
||||
)
|
||||
if kda_backend == "flashinfer":
|
||||
from tokenspeed_kernel.ops.attention.kda.flashinfer import (
|
||||
flashinfer_kda_supported,
|
||||
)
|
||||
|
||||
if not flashinfer_kda_supported():
|
||||
raise ValueError(
|
||||
"--kda-backend flashinfer requires an NVIDIA sm_100 or sm_103 device"
|
||||
)
|
||||
return kda_backend
|
||||
|
||||
|
||||
|
||||
@@ -3210,7 +3210,7 @@ class ServerArgs:
|
||||
parser.add_argument(
|
||||
"--kda-backend",
|
||||
type=str,
|
||||
choices=["auto", "fla", "flashkda", "cutedsl_kda"],
|
||||
choices=["auto", "fla", "flashkda", "cutedsl_kda", "flashinfer"],
|
||||
default=ServerArgs.kda_backend,
|
||||
help="KDA (Kimi Delta Attention) prefill kernel policy. On AMD, "
|
||||
"this setting is ignored and compatible kernels are selected using "
|
||||
@@ -3218,8 +3218,10 @@ class ServerArgs:
|
||||
"backend "
|
||||
"(cutedsl_kda > flashkda > fla). Named backends are NVIDIA-specific: "
|
||||
"'fla' uses the portable FLA scan, 'flashkda' uses the optional "
|
||||
"FlashKDA library (source build, SM90+), and 'cutedsl_kda' uses the "
|
||||
"CuteDSL KDA AOT kernel (prebuilt, sm_103a). Decode is unaffected.",
|
||||
"FlashKDA library (source build, SM90+), 'cutedsl_kda' uses the "
|
||||
"CuteDSL KDA AOT kernel (prebuilt, sm_103a), and 'flashinfer' uses "
|
||||
"FlashInfer's persistent CuTe DSL kernel (sm_100/sm_103, heads "
|
||||
"divisible by 8). Decode is unaffected.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--drafter-attention-backend",
|
||||
|
||||
@@ -1131,17 +1131,36 @@ def test_prefill_state_inputs_zero_fresh_rows_without_reading_null_page() -> Non
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("prefix", [0, 1024])
|
||||
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_kda_prefill_fused_staging_matches_pytorch(prefix, state_dtype, monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
("backend_name", "heads"), [("cutedsl_kda", 4), ("flashinfer", 8)]
|
||||
)
|
||||
def test_kda_prefill_fused_staging_matches_pytorch(
|
||||
prefix, state_dtype, backend_name, heads, monkeypatch
|
||||
):
|
||||
"""Native KDA outputs and persistent state match the old staging ops."""
|
||||
from tokenspeed_kernel.ops.attention.kda.cute_dsl import cutedsl_kda_supported
|
||||
from tokenspeed_kernel.ops.attention.kda.flashinfer import (
|
||||
flashinfer_kda_supported,
|
||||
)
|
||||
|
||||
if not cutedsl_kda_supported():
|
||||
pytest.skip("requires the native CuteDSL KDA prefill package")
|
||||
supported = {
|
||||
"cutedsl_kda": cutedsl_kda_supported,
|
||||
"flashinfer": flashinfer_kda_supported,
|
||||
}[backend_name]
|
||||
if not supported():
|
||||
pytest.skip(f"requires the {backend_name} KDA prefill kernel")
|
||||
# FlashInfer's persistent kernel takes head counts divisible by eight.
|
||||
monkeypatch.setattr(_KDAHarness, "H", heads)
|
||||
|
||||
contract = _stub_contract(prefix_granularity=1024, usable_pages=8)
|
||||
pools = [
|
||||
_StubContractPool(
|
||||
contract, "cuda", conv_dim=3 * 4 * 128, width=4, num_heads=4, head_dim=128
|
||||
contract,
|
||||
"cuda",
|
||||
conv_dim=3 * heads * 128,
|
||||
width=4,
|
||||
num_heads=heads,
|
||||
head_dim=128,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
@@ -1162,7 +1181,7 @@ def test_kda_prefill_fused_staging_matches_pytorch(prefix, state_dtype, monkeypa
|
||||
gid: [[2 * row + 1, 2 * row + 2]] for row, gid in enumerate(_STATE_GROUPS)
|
||||
}
|
||||
for harness in harnesses:
|
||||
harness.backend.kda_backend = "cutedsl_kda"
|
||||
harness.backend.kda_backend = backend_name
|
||||
harness.extend_metadata(
|
||||
tables,
|
||||
seq_lens=[prefix + 868],
|
||||
|
||||
@@ -6,6 +6,14 @@ requires FP32 storage. Native NVIDIA prefill scans receive FP32 state, so
|
||||
adapters widen BF16 checkpoints before calling their fixed ABI. State
|
||||
publication is the rounding boundary, not each recurrent arithmetic step.
|
||||
|
||||
The `flashinfer` prefill solution (sm100/sm103, head dim 128, heads divisible
|
||||
by eight, gate bound in [-5, 0)) runs FlashInfer's persistent CuTe DSL scan.
|
||||
It takes the BF16 gate and beta logits as produced, the V-major FP32 state the
|
||||
prefill gather yields, and advances that state in place. FlashInfer's public
|
||||
`recurrent_kda` plans this kernel from the device boundaries with a per-call
|
||||
`tolist()`, so the adapter plans from `cu_seqlens_cpu` and calls the kernel's
|
||||
forward entry, reusing one plan for every layer that shares the boundaries.
|
||||
|
||||
On sm100/sm103 (BF16 activations, head dim 128) speculative verify and replay
|
||||
are a raw-gate pair of persistent CuTe DSL kernels. The verify
|
||||
(`_cute_dsl/verify.py`) runs the gate GEMM straight into the replay payload,
|
||||
|
||||
@@ -811,6 +811,7 @@ def kda_replay_commit_supported(
|
||||
import tokenspeed_kernel.ops.attention.kda.triton # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.attention.kda.cuda # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.attention.kda.cute_dsl # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.attention.kda.flashinfer # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.attention.kda.gluon # noqa: E402,F401
|
||||
|
||||
# isort: on
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""FlashInfer persistent KDA prefill on SM100/SM103."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from tokenspeed_kernel.ops.attention.kda import KdaPrefillResult
|
||||
from tokenspeed_kernel.platform import (
|
||||
ArchVersion,
|
||||
CapabilityRequirement,
|
||||
current_platform,
|
||||
)
|
||||
from tokenspeed_kernel.registry import Priority, register_kernel
|
||||
from tokenspeed_kernel.signature import format_signatures
|
||||
from tokenspeed_kernel.thirdparty.flashinfer.kda import persistent_kda_prefill
|
||||
|
||||
_SUPPORTED_ARCHES = frozenset({ArchVersion(10, 0), ArchVersion(10, 3)})
|
||||
|
||||
|
||||
def flashinfer_kda_supported() -> bool:
|
||||
"""Whether this device runs FlashInfer's persistent KDA prefill kernel."""
|
||||
platform = current_platform()
|
||||
return platform.is_nvidia and platform.arch_version in _SUPPORTED_ARCHES
|
||||
|
||||
|
||||
@register_kernel(
|
||||
"attention",
|
||||
"kda_paged_prefill",
|
||||
name="flashinfer_kda_paged_prefill",
|
||||
solution="flashinfer",
|
||||
capability=CapabilityRequirement(
|
||||
min_arch_version=ArchVersion(10, 0),
|
||||
max_arch_version=ArchVersion(10, 3),
|
||||
vendors=frozenset({"nvidia"}),
|
||||
),
|
||||
signatures=format_signatures(("q", "k", "v"), "dense", {torch.bfloat16}),
|
||||
priority=Priority.SPECIALIZED,
|
||||
traits={"recurrent_layout": frozenset({"v_major"})},
|
||||
)
|
||||
def flashinfer_kda_paged_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g_raw: torch.Tensor,
|
||||
beta_logits: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
*,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
cu_seqlens_cpu: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
) -> KdaPrefillResult:
|
||||
"""Run the shared prefill contract with native V-major FP32 state.
|
||||
|
||||
Arguments follow kda_paged_prefill. The launch is planned from the host
|
||||
mirror ``cu_seqlens_cpu``; the device boundaries are not read. The kernel
|
||||
advances ``initial_state`` (the prefill state gather's private FP32 copy)
|
||||
in place and returns it as the final state.
|
||||
"""
|
||||
return KdaPrefillResult(
|
||||
*persistent_kda_prefill(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g_raw,
|
||||
beta_logits,
|
||||
A_log,
|
||||
dt_bias,
|
||||
initial_state=initial_state,
|
||||
cu_seqlens_cpu=cu_seqlens_cpu,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,166 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""FlashInfer KDA prefill integration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
|
||||
_HEAD_DIM = 128
|
||||
# Plans of recent boundary sets; every KDA layer of a forward shares one.
|
||||
_PLAN_CACHE_SIZE = 8
|
||||
_plans: OrderedDict[tuple, tuple] = OrderedDict()
|
||||
|
||||
|
||||
def _plan(offsets: tuple[int, ...], heads: int, device: torch.device) -> tuple:
|
||||
from flashinfer.kda_kernels.persistent.forward import make_plan
|
||||
|
||||
key = (offsets, heads, device)
|
||||
plan = _plans.get(key)
|
||||
if plan is None:
|
||||
plan = make_plan(list(offsets), heads, device)
|
||||
_plans[key] = plan
|
||||
if len(_plans) > _PLAN_CACHE_SIZE:
|
||||
_plans.popitem(last=False)
|
||||
else:
|
||||
_plans.move_to_end(key)
|
||||
return plan
|
||||
|
||||
|
||||
def persistent_kda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g_raw: torch.Tensor,
|
||||
beta_logits: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
*,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens_cpu: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Scan packed KDA prefill with FlashInfer's persistent SM100/SM103 kernel.
|
||||
|
||||
FlashInfer's public ``recurrent_kda`` plans this kernel from device
|
||||
boundaries with a ``tolist()`` per call, a stream-synchronizing D2H on
|
||||
every KDA layer. This adapter plans from the host mirror instead and
|
||||
calls the kernel's forward entry, so it repeats the public path's input
|
||||
contract here.
|
||||
|
||||
Args:
|
||||
q/k/v: Packed BF16 [1, tokens, heads, 128] activations; heads must be
|
||||
a multiple of eight.
|
||||
g_raw: Raw BF16 [1, tokens, heads, 128] gate; the decay gate is applied
|
||||
in-kernel.
|
||||
beta_logits: Raw BF16 [1, tokens, heads] beta projection.
|
||||
A_log/dt_bias: FP32 per-head decay [heads] and gate bias [heads * 128]
|
||||
or [heads, 128].
|
||||
initial_state: FP32 [sequences, heads, V, K] gathered cache state. The
|
||||
kernel advances it in place and returns it as the final state.
|
||||
cu_seqlens_cpu: Host int64 boundaries [sequences + 1], strictly
|
||||
increasing from 0 to the token count.
|
||||
lower_bound: Safe-gate bound in [-5, 0).
|
||||
|
||||
Returns:
|
||||
Packed BF16 output and the FP32 final state (``initial_state``).
|
||||
"""
|
||||
from flashinfer.kda_kernels.persistent.forward import fwd
|
||||
|
||||
if q.ndim != 4 or q.shape[0] != 1 or q.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
"FlashInfer persistent KDA requires packed BF16 [1, T, H, 128] q"
|
||||
)
|
||||
_, tokens, heads, dim = q.shape
|
||||
if dim != _HEAD_DIM or heads % 8:
|
||||
raise ValueError(
|
||||
f"FlashInfer persistent KDA requires head dim 128 and heads divisible "
|
||||
f"by 8; got heads={heads}, dim={dim}"
|
||||
)
|
||||
if lower_bound is None or not -5.0 <= float(lower_bound) < 0.0:
|
||||
raise ValueError("FlashInfer persistent KDA requires a lower bound in [-5, 0)")
|
||||
if tokens * heads * dim >= 2**31:
|
||||
raise ValueError("FlashInfer persistent KDA requires T*H*128 < 2**31")
|
||||
offsets = tuple(cu_seqlens_cpu.tolist())
|
||||
sequences = len(offsets) - 1
|
||||
if (
|
||||
sequences < 1
|
||||
or offsets[0] != 0
|
||||
or offsets[-1] != tokens
|
||||
or any(end <= start for start, end in zip(offsets, offsets[1:]))
|
||||
):
|
||||
raise ValueError(
|
||||
"KDA boundaries must rise strictly from 0 to the token count, "
|
||||
f"got {offsets[:4]}...{offsets[-1:]} for {tokens} tokens"
|
||||
)
|
||||
if sequences * heads * dim * dim >= 2**31:
|
||||
raise ValueError("FlashInfer persistent KDA requires N*H*128*128 < 2**31")
|
||||
if (
|
||||
initial_state.shape != (sequences, heads, dim, dim)
|
||||
or initial_state.dtype != torch.float32
|
||||
or not initial_state.is_contiguous()
|
||||
or initial_state.data_ptr() % 32
|
||||
):
|
||||
raise ValueError(
|
||||
"FlashInfer persistent KDA requires a contiguous, 32-byte-aligned FP32 "
|
||||
f"[{sequences}, {heads}, 128, 128] initial state"
|
||||
)
|
||||
if A_log.shape != (heads,) or A_log.dtype != torch.float32:
|
||||
raise ValueError("FlashInfer persistent KDA requires FP32 A_log [heads]")
|
||||
if dt_bias.dtype != torch.float32 or dt_bias.shape not in (
|
||||
(heads * dim,),
|
||||
(heads, dim),
|
||||
):
|
||||
raise ValueError(
|
||||
"FlashInfer persistent KDA requires FP32 dt_bias [heads * 128]"
|
||||
)
|
||||
# The kernel's descriptors index dense rows; serving's beta is a slice of
|
||||
# the merged projection, so it is the input this usually copies.
|
||||
q, k, v, g_raw, beta_logits, A_log, dt_bias = (
|
||||
t.contiguous() for t in (q, k, v, g_raw, beta_logits, A_log, dt_bias)
|
||||
)
|
||||
for name, tensor in (("k", k), ("v", v), ("g_raw", g_raw)):
|
||||
if tensor.shape != q.shape or tensor.dtype != torch.bfloat16:
|
||||
raise ValueError(f"FlashInfer persistent KDA requires BF16 {name} like q")
|
||||
if beta_logits.shape != q.shape[:3] or beta_logits.dtype != torch.bfloat16:
|
||||
raise ValueError("FlashInfer persistent KDA requires BF16 beta [1, T, H]")
|
||||
plan = _plan(offsets, heads, q.device)
|
||||
output = torch.empty_like(v)
|
||||
with torch.cuda.device(q.device):
|
||||
fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
g_raw,
|
||||
beta_logits,
|
||||
dim**-0.5,
|
||||
output,
|
||||
A_log,
|
||||
dt_bias,
|
||||
float(lower_bound),
|
||||
initial_state=initial_state,
|
||||
final_state=initial_state,
|
||||
cu_seqlens=plan[0],
|
||||
plan=plan,
|
||||
)
|
||||
return output, initial_state
|
||||
@@ -0,0 +1,154 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""FlashInfer persistent KDA prefill through kda_paged_prefill vs the portable FLA scan.
|
||||
|
||||
The FlashInfer solution keeps V-major [N, H, V, K] states while FLA uses
|
||||
[N, H, K, V]; the reference's states are converted at the comparison boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib.util import find_spec
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from tokenspeed_kernel.ops.attention.kda import kda_paged_prefill
|
||||
from tokenspeed_kernel.ops.attention.kda.flashinfer import flashinfer_kda_supported
|
||||
|
||||
HEADS = 8
|
||||
DIM = 128
|
||||
LOWER_BOUND = -5.0
|
||||
OUTPUT_MAX_ERROR = 2e-3
|
||||
STATE_MAX_ERROR = 5e-3
|
||||
|
||||
requires_flashinfer_kda = pytest.mark.skipif(
|
||||
not (
|
||||
torch.cuda.is_available()
|
||||
and find_spec("fla") is not None
|
||||
and flashinfer_kda_supported()
|
||||
),
|
||||
reason="requires an sm100/sm103 GPU and flash-linear-attention",
|
||||
)
|
||||
|
||||
|
||||
def _inputs(lengths: list[int], *, heads: int = HEADS, seed: int = 20261108):
|
||||
torch.manual_seed(seed)
|
||||
tokens = sum(lengths)
|
||||
q = torch.randn(1, tokens, heads, DIM, device="cuda").bfloat16()
|
||||
k = torch.randn(1, tokens, heads, DIM, device="cuda").bfloat16()
|
||||
v = (torch.randn(1, tokens, heads, DIM, device="cuda") * 0.1).bfloat16()
|
||||
g_raw = torch.randn(1, tokens, heads, DIM, device="cuda", dtype=torch.bfloat16)
|
||||
# Serving hands beta over as a slice of the merged projection.
|
||||
beta = torch.randn(1, tokens, heads + DIM, device="cuda", dtype=torch.bfloat16)[
|
||||
..., -heads:
|
||||
]
|
||||
A_log = torch.randn(heads, device="cuda", dtype=torch.float32) * 0.25
|
||||
dt_bias = (
|
||||
torch.randn(heads, DIM, device="cuda", dtype=torch.float32) * 1.4 - 4.6
|
||||
).clamp_(-9.0, -0.7)
|
||||
boundaries_cpu = torch.tensor([0, *lengths], dtype=torch.int64).cumsum(0)
|
||||
state = torch.randn(len(lengths), heads, DIM, DIM, device="cuda") * 0.05
|
||||
return (q, k, v, g_raw, beta, A_log, dt_bias), state, boundaries_cpu
|
||||
|
||||
|
||||
def _flashinfer(inputs, state, boundaries_cpu, boundaries=None):
|
||||
return kda_paged_prefill(
|
||||
*inputs,
|
||||
initial_state=state,
|
||||
cu_seqlens=boundaries_cpu.cuda() if boundaries is None else boundaries,
|
||||
cu_seqlens_cpu=boundaries_cpu,
|
||||
capacity=None,
|
||||
lower_bound=LOWER_BOUND,
|
||||
solution="flashinfer",
|
||||
recurrent_layout="v_major",
|
||||
)
|
||||
|
||||
|
||||
def _assert_matches_portable(lengths: list[int], fresh_rows: tuple[int, ...] = ()):
|
||||
from tokenspeed_kernel.ops.attention.kda._triton.fla import kda_chunk_prefill
|
||||
|
||||
inputs, state, boundaries_cpu = _inputs(lengths)
|
||||
for row in fresh_rows:
|
||||
state[row].zero_()
|
||||
expected_out, expected_state = kda_chunk_prefill(
|
||||
*inputs,
|
||||
initial_state=state.transpose(-1, -2).contiguous(),
|
||||
cu_seqlens=boundaries_cpu.cuda(),
|
||||
cu_seqlens_cpu=boundaries_cpu,
|
||||
lower_bound=LOWER_BOUND,
|
||||
beta_is_logit=True,
|
||||
)
|
||||
actual = _flashinfer(inputs, state.clone(), boundaries_cpu)
|
||||
torch.cuda.synchronize()
|
||||
out_err = (actual.out.float() - expected_out.float()).abs().max().item()
|
||||
state_err = (
|
||||
(actual.final_state.float() - expected_state.transpose(-1, -2).float())
|
||||
.abs()
|
||||
.max()
|
||||
.item()
|
||||
)
|
||||
assert out_err <= OUTPUT_MAX_ERROR, f"output max error {out_err}"
|
||||
assert state_err <= STATE_MAX_ERROR, f"state max error {state_err}"
|
||||
|
||||
|
||||
@requires_flashinfer_kda
|
||||
def test_unaligned_single_sequence_matches_portable() -> None:
|
||||
_assert_matches_portable([500])
|
||||
|
||||
|
||||
@requires_flashinfer_kda
|
||||
def test_fresh_state_matches_portable() -> None:
|
||||
_assert_matches_portable([512], fresh_rows=(0,))
|
||||
|
||||
|
||||
@requires_flashinfer_kda
|
||||
def test_varlen_batch_matches_portable() -> None:
|
||||
_assert_matches_portable([130, 517, 64], fresh_rows=(1,))
|
||||
|
||||
|
||||
@requires_flashinfer_kda
|
||||
@pytest.mark.parametrize("lengths", [[1], [1, 1, 3]])
|
||||
def test_single_token_sequences_match_portable(lengths) -> None:
|
||||
"""A fully cached prompt prefills one token; batches can hold only those."""
|
||||
_assert_matches_portable(lengths)
|
||||
|
||||
|
||||
@requires_flashinfer_kda
|
||||
def test_known_boundaries_issue_no_device_sync() -> None:
|
||||
"""Every KDA layer of a forward shares its boundaries: after the first
|
||||
layer plans them, a layer must not synchronize the launch stream."""
|
||||
inputs, state, boundaries_cpu = _inputs([700, 300])
|
||||
boundaries = boundaries_cpu.cuda()
|
||||
_flashinfer(inputs, state.clone(), boundaries_cpu, boundaries)
|
||||
layer_state = state.clone()
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
_flashinfer(inputs, layer_state, boundaries_cpu, boundaries)
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode("default")
|
||||
|
||||
|
||||
@requires_flashinfer_kda
|
||||
def test_rejects_heads_not_divisible_by_eight() -> None:
|
||||
inputs, state, boundaries_cpu = _inputs([64], heads=12)
|
||||
with pytest.raises(ValueError, match="heads divisible by 8"):
|
||||
_flashinfer(inputs, state, boundaries_cpu)
|
||||
@@ -694,6 +694,7 @@ def test_nvidia_kda_verify_and_decode_registration_traits(
|
||||
("triton_nvidia_kda_paged_prefill", "k_major"),
|
||||
("flashkda_nvidia_kda_paged_prefill", "k_major"),
|
||||
("cutedsl_kda_nvidia_paged_prefill", "v_major"),
|
||||
("flashinfer_kda_paged_prefill", "v_major"),
|
||||
],
|
||||
)
|
||||
def test_nvidia_kda_prefill_kernels_declare_native_layout(kernel_name, layout) -> None:
|
||||
|
||||
Reference in New Issue
Block a user