perf(dsa): select a KVP rank's owned top-k in one CuTe DSL launch (#3585)

## Summary

On SM100/SM103 no leaf served `dsa_select_owned_candidates`, so KVP
decode gathered every rank's (position, score) records and ran the merge
chain: a persistent radix top-k, the threshold selection, the global
slot mapping and the owned-slot compaction. That chain costs about 36-42
us per DSA layer.

- `cute_dsl_dsa_select_owned_candidates` is one launch with one
1024-thread CTA per row. The row's gathered scores stay in registers,
three radix digits over a shared-memory histogram find the k-th largest,
and two block scans write the rank's kept candidates as local slots in
candidate order.
- Its output is bit for bit the merge's selection compacted to the rank.
Only scores cross the group, half the bytes of the records.
- Models with `owned_decode_topk` (Kimi-K3.1) now take the owned path on
NVIDIA KVP decode, as they already did on gfx950.

## Test Plan

- [x]
`tokenspeed-kernel/test/nvidia/ops/attention/test_cute_dsl_dsa_owned_selection.py`:
11 passed on GB300. It checks every rank's rows against the merge
compacted to the rank, including mandatory, signed-zero, NaN, short and
empty rows, tied, narrow and single-value score bands, positions past
int32 offsets, one compile across batch rows, and refusals.
- [x] `test/runtime/distributed/test_dsa_dcp_owned_decode_topk.py` now
also runs on SM100/SM103: 2 and 4 ranks over NCCL passed, including
graph replays.
- [x] Kernel time, eight ranks, top-2048, 8-128 rows: 9.5-10.1 us vs
35.8-41.7 us for the merge chain, outputs identical.
- [x] Kimi-K3.1, TP8 with KVP8 on 2x GB300, DSpark with synthetic
acceptance 8. Each value is the mean of two fresh boots, in ms/token:

| batch | 1 | 2 | 4 | 8 | 16 | 32 | 64 |
|---|---|---|---|---|---|---|---|
| main | 1.577 | 1.897 | 2.440 | 3.145 | 4.577 | 6.042 | 8.171 |
| this PR | 1.529 | 1.862 | 2.386 | 3.077 | 4.519 | 5.745 | 7.949 |
| change | -3.1% | -1.8% | -2.2% | -2.2% | -1.3% | -4.9% | -2.7% |

---------

Signed-off-by: Nicolas Perrin <nperrin.ai@gmail.com>
This commit is contained in:
Nicolas Perrin
2026-11-18 14:32:09 -05:00
committed by GitHub
parent 58f7374c07
commit 5b26453b01
6 changed files with 806 additions and 16 deletions
@@ -113,6 +113,11 @@ def _page_table(placed: list[list[int]]) -> torch.Tensor:
def _plane(keys, virtual_blocks, placement, device):
"""This rank's FP8 index-K plane of every virtual slot's key."""
from tokenspeed_kernel.ops.kvcache.index_k_scale import (
encode_index_k_scale,
index_k_scale_ue8m0,
round_index_k_scale,
)
from tokenspeed_kernel.ops.kvcache.triton import index_k_block_split_scatter
from tokenspeed_kernel.ops.quantization import quantize_fp8
@@ -125,6 +130,12 @@ def _plane(keys, virtual_blocks, placement, device):
values, scales = quantize_fp8(
keys, granularity="token_group", group_size=128, scale_encoding="float32"
)
if index_k_scale_ue8m0():
# A Blackwell plane's keys quantize with the power-of-two scale its
# UE8M0 word stores, as the index-K writes do.
scales = round_index_k_scale(scales)
values = (keys.float() / scales).clamp(-448, 448).to(values.dtype)
scales = encode_index_k_scale(scales)
slots, owned = resolve_cache_slots(
torch.arange(keys.shape[0], device=device), placement
)
@@ -446,17 +457,25 @@ def _worker(rank, degree, rendezvous, pull_exchange):
dist.destroy_process_group()
# NCCL and the pull exchange at two ranks; the pull exchange at K3.1's eight.
@pytest.mark.parametrize("degree,pull_exchange", [(2, False), (2, True), (8, True)])
# NCCL at two and four ranks, the pull exchange at two and at K3.1's eight.
@pytest.mark.parametrize(
"degree,pull_exchange", [(2, False), (4, False), (2, True), (8, True)]
)
def test_owned_decode_topk_is_each_ranks_share_of_the_merge(
tmp_path, degree, pull_exchange
):
from tokenspeed_kernel.platform import current_platform
from tokenspeed_kernel.platform import ArchVersion, current_platform
if not torch.cuda.is_available() or torch.cuda.device_count() < degree:
pytest.skip(f"requires {degree} GPUs")
if not current_platform().is_cdna4:
pytest.skip("the owned selection and Iris pull exchanges run on CDNA4")
platform = current_platform()
if pull_exchange and not platform.is_cdna4:
pytest.skip("Iris pull exchanges run on CDNA4")
if not platform.is_cdna4 and not (
platform.is_nvidia
and ArchVersion(10, 0) <= platform.arch_version <= ArchVersion(10, 3)
):
pytest.skip("the owned selection runs on CDNA4, SM100, and SM103")
mp.spawn(
_worker,
args=(degree, f"file://{tmp_path / 'rendezvous'}", pull_exchange),
+19 -6
View File
@@ -133,6 +133,10 @@ def _selection_case(
from tokenspeed_kernel.platform import current_platform
from tokenspeed.runtime.layers.attention.dcp.placement import (
compact_owned_cache_slots,
)
sharded = _sharded_layer(
rank, kv_cache_dtype=kv_cache_dtype, spec_width=spec_width, index_topk=512
)
@@ -168,16 +172,25 @@ def _selection_case(
*whole, hidden, **run
)
if sharded[2].takes_owned_decode_topk(sharded[0].attn_mqa):
# Each rank keeps the selected keys it owns.
dist.all_reduce(counts)
assert torch.equal(counts, expected_counts)
assert (counts == 512).all()
# Each rank keeps the selected keys it owns, as local slots.
expected_slots, expected_counts = compact_owned_cache_slots(
expected_slots,
expected_counts,
sharded[2].cache_placement(sharded[0].attn_mqa),
)
total = counts.clone()
dist.all_reduce(total)
assert (total == 512).all()
else:
assert (counts == 512).all()
# gfx950 scores KVP candidates with the Triton leaf and one rank with Gluon's.
if current_platform().is_amd:
return
assert torch.equal(counts, expected_counts)
for row in range(spec_width):
assert sorted(slots[row, :512].tolist()) == sorted(
expected_slots[row, :512].tolist()
live = int(counts[row])
assert sorted(slots[row, :live].tolist()) == sorted(
expected_slots[row, :live].tolist()
)
for got, reference in zip(outputs, expected, strict=True):
_assert_close_to_reference(got, reference.float(), fraction)
@@ -157,11 +157,16 @@ row. That is bit for bit `cuda_dsa_merge_candidates`' selection compacted to
the rank's slots, without exchanging positions; `triton_dsa_merge_candidates`
selects the same candidates in score order. A score plane reads an invalid
candidate off its score, so `scores_out` holds -inf for every -1 offset.
Only gfx950 registers it, as one launch over the gathered planes:
`dsa_select_owned_candidates_supported` accepts a power-of-two rank count, a
power-of-two topk of at least 512, at most 16384 gathered scores per row and
rank chunks within 32-bit offsets, and refuses other platforms. A KVP decode
whose sizes it refuses merges the candidates instead.
gfx950 and SM100/SM103 register it for a topk of 512, 1024, or 2048, each as
one launch over the gathered planes. gfx950's leaf accepts a power-of-two rank
count, at most 16384 gathered scores per row, and rank chunks within 32-bit
offsets. `cute_dsl_dsa_select_owned_candidates` runs one 1024-thread CTA per
row: the row's scores stay in registers, three radix digits of 11, 11, and 10
bits over a shared-memory histogram find the k-th largest score, and two block
scans keep the rank's candidates in candidate order. It accepts two or more
power-of-two ranks and at most 32768 gathered scores per row.
`dsa_select_owned_candidates_supported` refuses other sizes and platforms; a
KVP decode whose sizes it refuses merges the candidates instead.
## Materialized Triton row top-k
@@ -0,0 +1,337 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Select a KVP rank's share of a merged top-k in one CTA per row.
The selection reads every rank's gathered candidate scores. Each CTA holds its row's ``ranks * topk`` scores as 32-bit order keys (NaN and
-inf as 0, -0 as +0), finds the k-th largest live key for k = ``topk`` (the
smallest one with fewer live) in three radix digits of 11, 11, and 10 bits, and
keeps this rank's candidates above it, then the ties at it through the ranks in
order and within a rank in candidate order, ``topk`` in all: this rank's share
of the merge's selection. Kept candidates map to local slots through the local
page table in candidate order, ``-1`` after.
"""
from __future__ import annotations
import functools
import cutlass
import cutlass.cute as cute
import torch
from cutlass import Int32, Uint32
from tokenspeed_kernel.thirdparty.cute_dsl.k3_decode.common.primitives import (
float_bits,
)
from tokenspeed_kernel.thirdparty.cute_dsl.k3_decode.dsa.indexer import (
BINS,
LOW_BINS,
SELECT_THREADS,
SELECT_WARPS,
block_exclusive,
order_key,
shared_add,
table_entry,
topdown,
warp_sum,
)
THREADS = SELECT_THREADS
# Every rank's scores of a row in registers, at most this many per thread.
MAX_KEYS_PER_THREAD = 32
NAN_BITS = 0x7F800000
NEG_INF_BITS = 0xFF800000
@cute.jit
def _live_key(bits: Uint32) -> Uint32:
"""Return the order key of FP32 ``bits``, or 0 for NaN and -inf.
Every live key is above 0, and -0 shares +0's key.
"""
key = order_key(bits)
if (bits & Uint32(0x7FFFFFFF)) > Uint32(NAN_BITS):
key = Uint32(0)
if bits == Uint32(NEG_INF_BITS):
key = Uint32(0)
return key
class OwnedSelect:
"""Select one rank's share of the merged top-k, one CTA per row.
Reads ``degree`` ranks' gathered FP32 ``[degree, rows, width]`` scores and
writes rank ``rank``'s local slots ``[rows, width]`` and counts ``[rows]``.
"""
def __init__(self, degree: int, width: int, page: int):
self.degree, self.width, self.page = degree, width, page
self.keys = degree * width // THREADS
self.own = max(1, width // THREADS)
@cute.jit
def __call__(
self,
scores: cute.Tensor,
positions: cute.Tensor,
table: cute.Tensor,
requests: cute.Tensor,
out: cute.Tensor,
lens: cute.Tensor,
rank: Int32,
stream,
):
self.kernel(scores, positions, table, requests, out, lens, rank).launch(
grid=(out.shape[0], 1, 1), block=(THREADS, 1, 1), stream=stream
)
@cute.jit
def digit(self, hist, info, keys, shift, mask, match_shift, match, tid):
"""Histogram one radix digit of the live keys under a prefix.
Counts ``(key >> shift) & mask`` over the live keys whose bits above
``match_shift`` are ``match`` (every live key at 32).
"""
cute.arch.sync_threads()
for b in cutlass.range(tid, BINS, THREADS, unroll=1):
hist[b] = Uint32(0)
if tid == 0:
info[2] = Int32(0)
info[3] = Int32(0)
cute.arch.sync_threads()
for i in cutlass.range_constexpr(self.keys):
key = keys[i]
hit = Int32(0)
if key != Uint32(0):
hit = Int32(1)
if cutlass.const_expr(match_shift < 32):
if (key >> Uint32(match_shift)) != match:
hit = Int32(0)
if hit != Int32(0):
shared_add(hist.iterator + ((key >> Uint32(shift)) & mask), 1)
cute.arch.sync_threads()
@cute.kernel
def kernel(self, scores, positions, table, requests, out, lens, rank):
"""Select rank ``rank``'s share of one row's merged top-k.
Reads the row's scores of every rank, finds the threshold key in three
radix digits, then keeps this rank's candidates above it and the ties
at it its quota allows (lower ranks' ties first, then candidate
order). Writes their local slots through ``positions`` and ``table``
to ``out`` in candidate order, ``-1`` after, and the count to ``lens``.
"""
W = self.width
tid, _, _ = cute.arch.thread_idx()
row, _, _ = cute.arch.block_idx()
warp = cute.arch.make_warp_uniform(tid // 32)
lane = tid % 32
smem = cutlass.memory.SmemAllocator()
hist = smem.allocate_tensor(Uint32, cute.make_layout(BINS), 16)
scan = smem.allocate_tensor(Uint32, cute.make_layout(SELECT_WARPS), 16)
info = smem.allocate_tensor(Int32, cute.make_layout(8), 16)
# Column c = tid + THREADS * i of the row: rank c // W, candidate c % W.
raw = []
for i in cutlass.range_constexpr(self.keys):
column = tid + THREADS * i
raw.append(scores[column // W, row, column % W])
keys = []
live = Uint32(0)
for i in cutlass.range_constexpr(self.keys):
key = _live_key(Uint32(float_bits(raw[i])))
if key != Uint32(0):
live += Uint32(1)
keys.append(key)
# This rank's candidates [tid * own, tid * own + own) in candidate
# order, with their local slots: loads that overlap the radix digits.
first = tid * self.own
request = requests[row]
own_keys = []
slots = []
for e in cutlass.range_constexpr(self.own):
key = Uint32(0)
slot = Int32(-1)
if first + e < W:
key = _live_key(Uint32(float_bits(scores[rank, row, first + e])))
if key != Uint32(0):
position = positions[row, first + e]
if position >= Int32(0):
page = table_entry(table, request, position // Int32(self.page))
if page >= Int32(0):
slot = page * Int32(self.page) + position % Int32(self.page)
own_keys.append(key)
slots.append(slot)
live = warp_sum(live)
if tid == 0:
info[4] = Int32(0)
cute.arch.sync_threads()
if lane == 0:
shared_add(info.iterator + 4, live)
# The threshold: three radix digits over the live keys.
self.digit(hist, info, keys, 21, 0x7FF, 32, Uint32(0), tid)
need = cutlass.min(Uint32(info[4]), Uint32(W))
topdown(hist, BINS, need, info, scan, tid, warp, lane)
c0 = Uint32(info[2])
need -= Uint32(info[3])
self.digit(hist, info, keys, 10, 0x7FF, 21, c0, tid)
topdown(hist, BINS, need, info, scan, tid, warp, lane)
c1 = Uint32(info[2])
need -= Uint32(info[3])
prefix = (c0 << Uint32(11)) | c1
self.digit(hist, info, keys, 0, 0x3FF, 10, prefix, tid)
topdown(hist, LOW_BINS, need, info, scan, tid, warp, lane)
threshold = (prefix << Uint32(10)) | Uint32(info[2])
# The ties at the threshold to take, through the ranks in order: one
# scan sums the lower ranks' ties (high half) and ranks this rank's.
take = need - Uint32(info[3])
ties = Uint32(0)
for i in cutlass.range_constexpr(self.keys):
column = tid + THREADS * i
if keys[i] == threshold and keys[i] != Uint32(0):
if column // W < rank:
ties += Uint32(1 << 16)
for e in cutlass.range_constexpr(self.own):
if own_keys[e] == threshold and own_keys[e] != Uint32(0):
ties += Uint32(1)
tie_rank, tie_total = block_exclusive(ties, scan, warp, lane)
tie_rank = (tie_rank & Uint32(0xFFFF)) + (tie_total >> Uint32(16))
kept = Uint32(0)
kept_slots = []
for e in cutlass.range_constexpr(self.own):
key = own_keys[e]
slot = slots[e]
keep = Int32(0)
if key > threshold:
keep = Int32(1)
if key == threshold and key != Uint32(0):
if tie_rank < take:
keep = Int32(1)
tie_rank += Uint32(1)
if keep == Int32(0):
slot = Int32(-1)
if slot >= Int32(0):
kept += Uint32(1)
kept_slots.append(slot)
at, count = block_exclusive(kept, scan, warp, lane)
for e in cutlass.range_constexpr(self.own):
if kept_slots[e] >= Int32(0):
out[row, Int32(at)] = kept_slots[e]
at += Uint32(1)
for c in cutlass.range(tid, W, THREADS, unroll=1):
if c >= Int32(count):
out[row, c] = Int32(-1)
if tid == 0:
lens[row] = Int32(count)
def _fake(dtype, shape, stride, align):
return cute.runtime.make_fake_tensor(
dtype, shape, stride=stride, assumed_align=align
)
@functools.cache
def compile_owned_select(degree: int, width: int, page: int):
"""Compile the owned selection for a rank count, width, and page size.
Selects among ``degree`` ranks' ``width`` candidates per row over
``page``-token pages. Rows, the rank stride, the table, and the rank are
runtime values: one compile serves every batch.
"""
sym = cute.sym_int
scores = _fake(
cutlass.Float32, (degree, sym(), width), (cute.sym_int64(), width, 1), 4
)
positions = _fake(Int32, (sym(), width), (cute.sym_int64(), 1), 4)
table = _fake(Int32, (sym(), sym()), (cute.sym_int64(), 1), 4)
requests = _fake(Int32, (sym(),), (1,), 4)
out = _fake(Int32, (sym(), width), (width, 1), 4)
lens = _fake(Int32, (sym(),), (1,), 4)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
OwnedSelect(degree, width, page),
scores,
positions,
table,
requests,
out,
lens,
Int32(0),
stream,
options="--enable-tvm-ffi",
)
def owned_selection_supported(
*, ranks: int, rows: int, topk: int, rank_stride: int
) -> bool:
"""Return whether the owned selection takes these gathered scores.
The scores are ``ranks`` ranks' ``rows`` rows of ``topk`` candidates,
``rank_stride`` elements apart. It takes a power-of-two ``topk`` of at
least 512 and a power-of-two rank count whose every-rank row fits the
CTA's registers. Host values only.
"""
keys = ranks * topk
return (
topk >= 512
and topk & (topk - 1) == 0
and ranks >= 2
and ranks & (ranks - 1) == 0
and keys % THREADS == 0
and keys // THREADS <= MAX_KEYS_PER_THREAD
and rank_stride >= rows * topk
)
def cute_dsl_select_owned_from_gathered(
scores: torch.Tensor,
positions: torch.Tensor,
local_page_table: torch.Tensor,
query_requests: torch.Tensor,
*,
rank: int,
topk: int,
page_size: int,
out: torch.Tensor,
lens_out: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Select rank ``rank``'s share of the merged top-k in one launch.
Reads every rank's gathered FP32 ``[ranks, rows, topk]`` scores (rows
contiguous) and writes int32 ``out`` ``[rows, topk]``, the kept local slots
first in candidate order and ``-1`` after, and ``lens_out`` ``[rows]``
counts.
"""
degree, rows, width = scores.shape
if rows:
compile_owned_select(degree, width, page_size)(
scores,
positions,
local_page_table,
query_requests,
out,
lens_out,
rank,
)
return out, lens_out
@@ -55,6 +55,7 @@ from tokenspeed_kernel.platform import (
pdl_enabled,
)
from tokenspeed_kernel.registry import Priority, register_kernel
from tokenspeed_kernel.selection import traits_problem_filter
from tokenspeed_kernel.signature import dense_tensor_format, format_signature
__all__ = [
@@ -783,3 +784,127 @@ if _decode_front is not None:
float(softmax_scale),
)
return q_norm, projected, q_fp8, weights
# Owned selection: a KVP rank's share of the merged top-k in one launch.
if _fused_indexer is not None:
from tokenspeed_kernel.ops.attention.dsa._cute_dsl.owned_selection import (
cute_dsl_select_owned_from_gathered,
owned_selection_supported,
)
@register_kernel(
"attention",
"dsa_select_owned_candidates",
name="cute_dsl_dsa_select_owned_candidates",
solution="cute_dsl",
capability=CapabilityRequirement(
min_arch_version=ArchVersion(10, 0),
max_arch_version=ArchVersion(10, 3),
vendors=frozenset({"nvidia"}),
),
signatures=frozenset(
{format_signature(scores=dense_tensor_format(torch.float32))}
),
traits={
"topk": frozenset({512, 1024, 2048}),
"problem_filter": frozenset(
{
traits_problem_filter(
owned_selection_supported,
"ranks",
"rows",
"topk",
"rank_stride",
)
}
),
},
priority=Priority.SPECIALIZED,
)
def cute_dsl_dsa_select_owned_candidates(
scores: torch.Tensor,
positions: torch.Tensor,
local_page_table: torch.Tensor,
query_requests: torch.Tensor,
*,
rank: int,
topk: int,
page_size: int,
out: torch.Tensor | None = None,
lens_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run ``dsa_select_owned_candidates`` in one launch.
Serves the sizes ``dsa_select_owned_candidates_supported`` admits.
Args:
scores: FP32 ``[ranks, rows, topk]`` every rank's gathered
candidate scores (``-inf`` invalid), rows contiguous; the rank
stride may be wider.
positions: Int32 ``[rows, topk]`` logical positions of this
rank's candidates, rows contiguous.
local_page_table: Int32 ``[requests, pages]`` this rank's local
pages (``-1`` where not owned), rows contiguous.
query_requests: Int32 ``[rows]`` request of every row.
rank: This rank's index in ``scores``.
topk: Entries selected per row over all ranks.
page_size: Tokens per page of ``local_page_table``.
out: Optional contiguous int32 ``[rows, topk]`` output.
lens_out: Optional contiguous int32 ``[rows]`` output.
Returns:
``(out, lens_out)``: this rank's kept candidates as local slots
in candidate order, ``-1`` after, and their count per row.
"""
degree, rows, width = scores.shape
if (
width != topk
or scores.stride()[1:] != (width, 1)
or not owned_selection_supported(
ranks=degree, rows=rows, topk=topk, rank_stride=scores.stride(0)
)
):
raise ValueError(
"owned selection needs FP32 [ranks, rows, topk] scores with "
"contiguous rows, power-of-two ranks and topk, and every rank's "
f"row in registers; got {tuple(scores.shape)} strided "
f"{scores.stride()} for top-{topk}"
)
if not 0 <= rank < degree:
raise ValueError(f"rank {rank} is outside {degree} ranks")
if any(
value.dtype != torch.int32
for value in (positions, local_page_table, query_requests)
):
raise TypeError(
"candidate positions, local pages, and requests must be int32"
)
if positions.shape != (rows, width) or positions.stride(1) != 1:
raise ValueError(
"this rank's candidate positions need contiguous [rows, topk] rows"
)
if local_page_table.stride(1) != 1:
raise ValueError("local pages need contiguous rows")
if query_requests.shape != (rows,) or not query_requests.is_contiguous():
raise ValueError("every row needs its request")
if out is None:
out = torch.empty((rows, topk), device=scores.device, dtype=torch.int32)
if lens_out is None:
lens_out = torch.empty((rows,), device=scores.device, dtype=torch.int32)
if out.shape != (rows, topk) or not out.is_contiguous():
raise ValueError("owned slots need a contiguous [rows, topk] output")
if lens_out.shape != (rows,) or not lens_out.is_contiguous():
raise ValueError("owned counts need a contiguous [rows] output")
return cute_dsl_select_owned_from_gathered(
scores,
positions,
local_page_table,
query_requests,
rank=rank,
topk=topk,
page_size=page_size,
out=out,
lens_out=lens_out,
)
@@ -0,0 +1,291 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Check the CuTe DSL owned selection against the compacted merge.
The reference is the merge's selection compacted to each rank's owned slots.
Every rank's rows must match bit for bit, with mandatory, signed zero, NaN,
short and empty rows and score bands that stress the radix digits and the tie
quota; then positions past int32 offsets, one compile across batch rows, and
the sizes it refuses.
"""
from __future__ import annotations
import pytest
import torch
from utils import assert_no_cute_compile, is_nvidia
if not is_nvidia():
pytest.skip("requires an NVIDIA GPU", allow_module_level=True)
pytest.importorskip("cutlass")
from tokenspeed_kernel import platform # noqa: E402
if (
not platform.ArchVersion(10, 0)
<= platform.current_platform().arch_version
<= platform.ArchVersion(10, 3)
):
pytest.skip("requires SM100 or SM103", allow_module_level=True)
from tokenspeed_kernel.ops.attention.dsa import ( # noqa: E402
dsa_select_owned_candidates,
dsa_select_owned_candidates_supported,
)
from tokenspeed_kernel.ops.attention.dsa._triton.candidate_exchange import ( # noqa: E402
rank_major_scores,
select_through_threshold,
)
from tokenspeed_kernel.ops.attention.dsa.triton import ( # noqa: E402
local_topk_to_global_slots,
)
from tokenspeed_kernel.ops.kvcache.triton_cache_placement import ( # noqa: E402
compact_owned_slots,
virtual_slots_to_local,
)
_PAGE, _BLOCK = 64, 128
def _owned_candidates(degree: int, rows: int, q_len: int, topk: int, seed: int):
"""Build a virtual index-page table and every rank's owned candidates.
Returns positions ``[degree, rows, topk]`` (``-1`` invalid) on pages the
rank owns and their scores (``-inf`` invalid), with mandatory,
signed-zero, NaN, short, and empty rows.
"""
generator = torch.Generator(device="cuda").manual_seed(seed)
requests = rows // q_len
blocks_per_request = 24 * degree
virtual_blocks = requests * blocks_per_request + 1
blocks = 1 + torch.randperm(
virtual_blocks - 1, device="cuda", generator=generator
).view(requests, blocks_per_request)
pages_per_block = _BLOCK // _PAGE
table = blocks.unsqueeze(-1) * pages_per_block + torch.arange(
pages_per_block, device="cuda"
)
table = table.view(requests, -1).int()
owner = (table // pages_per_block - 1) % degree
positions = torch.full((degree, rows, topk), -1, device="cuda", dtype=torch.int32)
for source in range(degree):
for row in range(rows):
pages = (owner[row // q_len] == source).nonzero().flatten()
tokens = pages.unsqueeze(1) * _PAGE + torch.arange(_PAGE, device="cuda")
tokens = tokens.flatten()
pick = torch.randperm(tokens.numel(), device="cuda", generator=generator)
pick = tokens[pick[:topk]]
positions[source, row, : pick.numel()] = pick.int()
scores = torch.rand((degree, rows, topk), device="cuda", generator=generator)
scores[:, 0, 3:6] = torch.tensor([torch.inf, torch.inf, -0.0], device="cuda")
scores[:, 0, 6] = 0.0
scores[:, 0, 7] = torch.nan
if rows > 2:
# A short history: fewer valid candidates in all than topk.
positions[:, 1, topk // (2 * degree) :] = -1
# A request step whose history no rank owns.
positions[:, 2] = -1
scores[positions < 0] = -torch.inf
return table, virtual_blocks, positions, scores
def _local_pages(table, virtual_blocks: int, degree: int, rank: int):
slots, owned = virtual_slots_to_local(
table.long() * _PAGE,
rows_per_page=_BLOCK,
virtual_block_count=virtual_blocks,
degree=degree,
rank=rank,
)
return torch.where(owned, slots // _PAGE, -1).int()
def _merged_then_compacted(table, virtual_blocks, positions, scores, q_len, topk):
"""Return every rank's local slots through the full exchange.
That is the merge's selection (``cuda_dsa_merge_candidates``' threshold
selection over an exact top-k), its scheduler slots, then each rank's
owned share.
"""
degree = positions.shape[0]
records = torch.stack((positions, scores.view(torch.int32)), dim=-1)
merged = rank_major_scores(records)
order = torch.topk(merged, topk, dim=-1).indices
selected, counts = select_through_threshold(records, merged, order, topk=topk)
slots, lens = local_topk_to_global_slots(
local_topk_offsets=selected,
block_table=table,
block_size=_PAGE,
q_len_per_req=q_len,
)
torch.testing.assert_close(lens, counts, atol=0, rtol=0)
return [
compact_owned_slots(
slots,
lens,
rows_per_page=_BLOCK,
virtual_block_count=virtual_blocks,
degree=degree,
rank=rank,
)
for rank in range(degree)
]
# Eight ranks, 16 requests of 4 verify rows, top-2048: K3.1's KVP8 decode.
# The score bands that stress the radix passes and the tie quota run there.
@pytest.mark.parametrize(
"degree,rows,q_len,topk,band",
[(2, 64, 4, 512, "uniform"), (4, 64, 4, 2048, "uniform")]
+ [(8, 64, 4, topk, "uniform") for topk in (512, 2048)]
+ [(8, 7, 1, 2048, "uniform")]
+ [(8, 64, 4, 2048, band) for band in ("tied", "narrow", "single")],
)
def test_owned_selection_is_each_ranks_share_of_the_merge(
degree, rows, q_len, topk, band
):
table, virtual_blocks, positions, scores = _owned_candidates(
degree, rows, q_len, topk, seed=degree * 1000 + rows + topk
)
if band == "tied":
# The cutoff falls inside a tied group that spans the ranks.
generator = torch.Generator(device="cuda").manual_seed(rows + topk)
ties = torch.randint(
0, 8, scores.shape, device="cuda", generator=generator
).float()
scores = torch.where(positions < 0, scores, ties)
elif band == "narrow":
# Scores sharing their sign, exponent, and leading mantissa bits,
# beside the mandatory, zero, and NaN ones.
scores = torch.where(
scores.isfinite() & (scores > 0), 5 + scores * 1e-3, scores
)
elif band == "single":
scores = torch.where(positions < 0, scores, 2.5)
expected = _merged_then_compacted(
table, virtual_blocks, positions, scores, q_len, topk
)
# Each rank's shard of a send buffer wider than the scores.
gathered = torch.full(
(degree, rows * topk + 32), torch.nan, device="cuda", dtype=torch.float32
)
gathered[:, : rows * topk] = scores.view(degree, -1)
gathered = gathered[:, : rows * topk].view(degree, rows, topk)
requests = (torch.arange(rows, device="cuda") // q_len).int()
for rank, (slots, counts) in enumerate(expected):
local_slots, local_counts = dsa_select_owned_candidates(
gathered,
positions[rank].contiguous(),
_local_pages(table, virtual_blocks, degree, rank),
requests,
rank=rank,
topk=topk,
page_size=_PAGE,
solution="cute_dsl",
)
torch.testing.assert_close(local_slots, slots, atol=0, rtol=0)
torch.testing.assert_close(local_counts, counts, atol=0, rtol=0)
total = sum(counts for _, counts in expected)
assert (total <= topk).all()
assert (total[2] == 0).all()
def test_owned_selection_reads_positions_past_int32_offsets():
# This rank's candidate positions in rows an int32 stride apart, the last
# row (one with candidates) past 2**31 elements, selected into the
# caller's rows of wider buffers.
degree, rows, topk = 2, 4, 2048
table, virtual_blocks, positions, scores = _owned_candidates(
degree, rows, 1, topk, seed=23
)
(slots, counts), *_ = _merged_then_compacted(
table, virtual_blocks, positions, scores, 1, topk
)
assert int(counts[-1]) > 0
stride = (2**31 // (rows - 1) + 64) // 64 * 64
storage = torch.empty((rows - 1) * stride + topk, device="cuda", dtype=torch.int32)
far = storage.as_strided((rows, topk), (stride, 1))
far.copy_(positions[0])
assert (rows - 1) * far.stride(0) > 2**31
out = torch.full((rows + 2, topk), 7, device="cuda", dtype=torch.int32)
lens_out = torch.full((rows + 2,), 7, device="cuda", dtype=torch.int32)
local_slots, _ = dsa_select_owned_candidates(
scores,
far,
_local_pages(table, virtual_blocks, degree, 0),
torch.arange(rows, device="cuda", dtype=torch.int32),
rank=0,
topk=topk,
page_size=_PAGE,
solution="cute_dsl",
out=out[1 : rows + 1],
lens_out=lens_out[1 : rows + 1],
)
assert local_slots.data_ptr() == out[1].data_ptr()
torch.testing.assert_close(out[1 : rows + 1], slots, atol=0, rtol=0)
torch.testing.assert_close(lens_out[1 : rows + 1], counts, atol=0, rtol=0)
assert (out[0] == 7).all() and (out[rows + 1] == 7).all()
assert int(lens_out[0]) == 7 and int(lens_out[rows + 1]) == 7
def test_owned_selection_compiles_once_across_batch_rows():
def select(rows):
table, virtual_blocks, positions, scores = _owned_candidates(
8, rows, 1, 2048, seed=rows
)
dsa_select_owned_candidates(
scores,
positions[1].contiguous(),
_local_pages(table, virtual_blocks, 8, 1),
torch.arange(rows, device="cuda", dtype=torch.int32),
rank=1,
topk=2048,
page_size=_PAGE,
solution="cute_dsl",
)
select(1)
with assert_no_cute_compile():
for rows in (3, 8, 37, 64):
select(rows)
def test_owned_selection_refuses_gathered_scores_it_cannot_take():
assert dsa_select_owned_candidates_supported(
ranks=8, rows=64, topk=2048, rank_stride=64 * 2048 + 1152
)
# Such KVP decodes keep the merge.
for ranks, topk, rank_stride in (
# Not a power of two.
(3, 2048, 2048),
# Every rank's share of a row past the registers.
(32, 2048, 2048),
# A top-k under the CTA's threads' minimum.
(2, 256, 256),
# Rank chunks that overlap.
(8, 2048, 2047),
):
assert not dsa_select_owned_candidates_supported(
ranks=ranks, rows=1, topk=topk, rank_stride=rank_stride
)