perf(dsa): keep every live candidate of a short row without the radix digits (#3646)

## Summary

When a row's live candidates over every rank number at most the top-k (a
history no longer than it), the owned selection keeps all of them. The
CTA already sums the live count; it now skips the three radix digits and
the tie ranking in that case, with the same output.

## Test Plan

- [x]
`tokenspeed-kernel/test/nvidia/ops/attention/test_cute_dsl_dsa_owned_selection.py`:
12 passed on GB300, including a new case where every row fits the top-k.
- [x] One KVP8 rank's selection over a 1K-token history: 8 rows 8.8 ->
4.4 us, 64 rows 8.9 -> 4.5 us, the same slots and counts; rows of 2048
live candidates per rank unchanged (9.5 / 9.7 us).

---------

Signed-off-by: Nicolas Perrin <nperrin.ai@gmail.com>
This commit is contained in:
Nicolas Perrin
2026-11-19 11:21:31 -05:00
committed by GitHub
parent 1df4f4ba28
commit 5884085105
2 changed files with 57 additions and 30 deletions
@@ -26,7 +26,9 @@ smallest one with fewer live) in three radix digits of 11, 11, and 10 bits, and
keeps this rank's candidates above it, then the ties at it through the ranks in
order and within a rank in candidate order, ``topk`` in all: this rank's share
of the merge's selection. Kept candidates map to local slots through the local
page table in candidate order, ``-1`` after.
page table in candidate order, ``-1`` after. A row with at most ``topk`` live
keys in all (a history no longer than the top-k) keeps every one without the
digits.
"""
from __future__ import annotations
@@ -128,6 +130,45 @@ class OwnedSelect:
shared_add(hist.iterator + ((key >> Uint32(shift)) & mask), 1)
cute.arch.sync_threads()
@cute.jit
def threshold(self, hist, info, scan, keys, own_keys, rank, tid, warp, lane):
"""Find the row's threshold key and this rank's tie quota at it.
Three radix digits run over the row's live keys (more than ``W``
whenever this runs) to find the ``W``-th largest. Returns the threshold,
the ties to take at it, and this thread's first tie's rank among them
(lower ranks' ties first, then candidate order).
"""
W = self.width
self.digit(hist, info, keys, 21, 0x7FF, 32, Uint32(0), tid)
need = cutlass.min(Uint32(info[4]), Uint32(W))
topdown(hist, BINS, need, info, scan, tid, warp, lane)
c0 = Uint32(info[2])
need -= Uint32(info[3])
self.digit(hist, info, keys, 10, 0x7FF, 21, c0, tid)
topdown(hist, BINS, need, info, scan, tid, warp, lane)
c1 = Uint32(info[2])
need -= Uint32(info[3])
prefix = (c0 << Uint32(11)) | c1
self.digit(hist, info, keys, 0, 0x3FF, 10, prefix, tid)
topdown(hist, LOW_BINS, need, info, scan, tid, warp, lane)
threshold = (prefix << Uint32(10)) | Uint32(info[2])
# The ties at the threshold to take, through the ranks in order: one
# scan sums the lower ranks' ties (high half) and ranks this rank's.
take = need - Uint32(info[3])
ties = Uint32(0)
for i in cutlass.range_constexpr(self.keys):
column = tid + THREADS * i
if keys[i] == threshold and keys[i] != Uint32(0):
if column // W < rank:
ties += Uint32(1 << 16)
for e in cutlass.range_constexpr(self.own):
if own_keys[e] == threshold and own_keys[e] != Uint32(0):
ties += Uint32(1)
tie_rank, tie_total = block_exclusive(ties, scan, warp, lane)
tie_rank = (tie_rank & Uint32(0xFFFF)) + (tie_total >> Uint32(16))
return threshold, take, tie_rank
@cute.kernel
def kernel(self, scores, positions, table, requests, out, lens, rank):
"""Select rank ``rank``'s share of one row's merged top-k.
@@ -186,35 +227,17 @@ class OwnedSelect:
cute.arch.sync_threads()
if lane == 0:
shared_add(info.iterator + 4, live)
cute.arch.sync_threads()
# The threshold: three radix digits over the live keys.
self.digit(hist, info, keys, 21, 0x7FF, 32, Uint32(0), tid)
need = cutlass.min(Uint32(info[4]), Uint32(W))
topdown(hist, BINS, need, info, scan, tid, warp, lane)
c0 = Uint32(info[2])
need -= Uint32(info[3])
self.digit(hist, info, keys, 10, 0x7FF, 21, c0, tid)
topdown(hist, BINS, need, info, scan, tid, warp, lane)
c1 = Uint32(info[2])
need -= Uint32(info[3])
prefix = (c0 << Uint32(11)) | c1
self.digit(hist, info, keys, 0, 0x3FF, 10, prefix, tid)
topdown(hist, LOW_BINS, need, info, scan, tid, warp, lane)
threshold = (prefix << Uint32(10)) | Uint32(info[2])
# The ties at the threshold to take, through the ranks in order: one
# scan sums the lower ranks' ties (high half) and ranks this rank's.
take = need - Uint32(info[3])
ties = Uint32(0)
for i in cutlass.range_constexpr(self.keys):
column = tid + THREADS * i
if keys[i] == threshold and keys[i] != Uint32(0):
if column // W < rank:
ties += Uint32(1 << 16)
for e in cutlass.range_constexpr(self.own):
if own_keys[e] == threshold and own_keys[e] != Uint32(0):
ties += Uint32(1)
tie_rank, tie_total = block_exclusive(ties, scan, warp, lane)
tie_rank = (tie_rank & Uint32(0xFFFF)) + (tie_total >> Uint32(16))
# At most ``W`` live keys in all (a row no longer than the top-k): every
# live key is kept, so threshold 0 with no ties gives the same selection.
threshold = Uint32(0)
take = Uint32(0)
tie_rank = Uint32(0)
if Uint32(info[4]) > Uint32(W):
threshold, take, tie_rank = self.threshold(
hist, info, scan, keys, own_keys, rank, tid, warp, lane
)
kept = Uint32(0)
kept_slots = []
for e in cutlass.range_constexpr(self.own):
@@ -156,7 +156,7 @@ def _merged_then_compacted(table, virtual_blocks, positions, scores, q_len, topk
[(2, 64, 4, 512, "uniform"), (4, 64, 4, 2048, "uniform")]
+ [(8, 64, 4, topk, "uniform") for topk in (512, 2048)]
+ [(8, 7, 1, 2048, "uniform")]
+ [(8, 64, 4, 2048, band) for band in ("tied", "narrow", "single")],
+ [(8, 64, 4, 2048, band) for band in ("tied", "narrow", "single", "short")],
)
def test_owned_selection_is_each_ranks_share_of_the_merge(
degree, rows, q_len, topk, band
@@ -179,6 +179,10 @@ def test_owned_selection_is_each_ranks_share_of_the_merge(
)
elif band == "single":
scores = torch.where(positions < 0, scores, 2.5)
elif band == "short":
# Every row's history fits the top-k: each rank keeps all it holds.
positions[:, :, topk // (2 * degree) :] = -1
scores[positions < 0] = -torch.inf
expected = _merged_then_compacted(
table, virtual_blocks, positions, scores, q_len, topk
)