mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
perf(dsa): select a KVP rank's owned top-k in one CuTe DSL launch (#3585)
## Summary On SM100/SM103 no leaf served `dsa_select_owned_candidates`, so KVP decode gathered every rank's (position, score) records and ran the merge chain: a persistent radix top-k, the threshold selection, the global slot mapping and the owned-slot compaction. That chain costs about 36-42 us per DSA layer. - `cute_dsl_dsa_select_owned_candidates` is one launch with one 1024-thread CTA per row. The row's gathered scores stay in registers, three radix digits over a shared-memory histogram find the k-th largest, and two block scans write the rank's kept candidates as local slots in candidate order. - Its output is bit for bit the merge's selection compacted to the rank. Only scores cross the group, half the bytes of the records. - Models with `owned_decode_topk` (Kimi-K3.1) now take the owned path on NVIDIA KVP decode, as they already did on gfx950. ## Test Plan - [x] `tokenspeed-kernel/test/nvidia/ops/attention/test_cute_dsl_dsa_owned_selection.py`: 11 passed on GB300. It checks every rank's rows against the merge compacted to the rank, including mandatory, signed-zero, NaN, short and empty rows, tied, narrow and single-value score bands, positions past int32 offsets, one compile across batch rows, and refusals. - [x] `test/runtime/distributed/test_dsa_dcp_owned_decode_topk.py` now also runs on SM100/SM103: 2 and 4 ranks over NCCL passed, including graph replays. - [x] Kernel time, eight ranks, top-2048, 8-128 rows: 9.5-10.1 us vs 35.8-41.7 us for the merge chain, outputs identical. - [x] Kimi-K3.1, TP8 with KVP8 on 2x GB300, DSpark with synthetic acceptance 8. Each value is the mean of two fresh boots, in ms/token: | batch | 1 | 2 | 4 | 8 | 16 | 32 | 64 | |---|---|---|---|---|---|---|---| | main | 1.577 | 1.897 | 2.440 | 3.145 | 4.577 | 6.042 | 8.171 | | this PR | 1.529 | 1.862 | 2.386 | 3.077 | 4.519 | 5.745 | 7.949 | | change | -3.1% | -1.8% | -2.2% | -2.2% | -1.3% | -4.9% | -2.7% | --------- Signed-off-by: Nicolas Perrin <nperrin.ai@gmail.com>
This commit is contained in:
@@ -113,6 +113,11 @@ def _page_table(placed: list[list[int]]) -> torch.Tensor:
|
||||
|
||||
def _plane(keys, virtual_blocks, placement, device):
|
||||
"""This rank's FP8 index-K plane of every virtual slot's key."""
|
||||
from tokenspeed_kernel.ops.kvcache.index_k_scale import (
|
||||
encode_index_k_scale,
|
||||
index_k_scale_ue8m0,
|
||||
round_index_k_scale,
|
||||
)
|
||||
from tokenspeed_kernel.ops.kvcache.triton import index_k_block_split_scatter
|
||||
from tokenspeed_kernel.ops.quantization import quantize_fp8
|
||||
|
||||
@@ -125,6 +130,12 @@ def _plane(keys, virtual_blocks, placement, device):
|
||||
values, scales = quantize_fp8(
|
||||
keys, granularity="token_group", group_size=128, scale_encoding="float32"
|
||||
)
|
||||
if index_k_scale_ue8m0():
|
||||
# A Blackwell plane's keys quantize with the power-of-two scale its
|
||||
# UE8M0 word stores, as the index-K writes do.
|
||||
scales = round_index_k_scale(scales)
|
||||
values = (keys.float() / scales).clamp(-448, 448).to(values.dtype)
|
||||
scales = encode_index_k_scale(scales)
|
||||
slots, owned = resolve_cache_slots(
|
||||
torch.arange(keys.shape[0], device=device), placement
|
||||
)
|
||||
@@ -446,17 +457,25 @@ def _worker(rank, degree, rendezvous, pull_exchange):
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
# NCCL and the pull exchange at two ranks; the pull exchange at K3.1's eight.
|
||||
@pytest.mark.parametrize("degree,pull_exchange", [(2, False), (2, True), (8, True)])
|
||||
# NCCL at two and four ranks, the pull exchange at two and at K3.1's eight.
|
||||
@pytest.mark.parametrize(
|
||||
"degree,pull_exchange", [(2, False), (4, False), (2, True), (8, True)]
|
||||
)
|
||||
def test_owned_decode_topk_is_each_ranks_share_of_the_merge(
|
||||
tmp_path, degree, pull_exchange
|
||||
):
|
||||
from tokenspeed_kernel.platform import current_platform
|
||||
from tokenspeed_kernel.platform import ArchVersion, current_platform
|
||||
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < degree:
|
||||
pytest.skip(f"requires {degree} GPUs")
|
||||
if not current_platform().is_cdna4:
|
||||
pytest.skip("the owned selection and Iris pull exchanges run on CDNA4")
|
||||
platform = current_platform()
|
||||
if pull_exchange and not platform.is_cdna4:
|
||||
pytest.skip("Iris pull exchanges run on CDNA4")
|
||||
if not platform.is_cdna4 and not (
|
||||
platform.is_nvidia
|
||||
and ArchVersion(10, 0) <= platform.arch_version <= ArchVersion(10, 3)
|
||||
):
|
||||
pytest.skip("the owned selection runs on CDNA4, SM100, and SM103")
|
||||
mp.spawn(
|
||||
_worker,
|
||||
args=(degree, f"file://{tmp_path / 'rendezvous'}", pull_exchange),
|
||||
|
||||
@@ -133,6 +133,10 @@ def _selection_case(
|
||||
|
||||
from tokenspeed_kernel.platform import current_platform
|
||||
|
||||
from tokenspeed.runtime.layers.attention.dcp.placement import (
|
||||
compact_owned_cache_slots,
|
||||
)
|
||||
|
||||
sharded = _sharded_layer(
|
||||
rank, kv_cache_dtype=kv_cache_dtype, spec_width=spec_width, index_topk=512
|
||||
)
|
||||
@@ -168,16 +172,25 @@ def _selection_case(
|
||||
*whole, hidden, **run
|
||||
)
|
||||
if sharded[2].takes_owned_decode_topk(sharded[0].attn_mqa):
|
||||
# Each rank keeps the selected keys it owns.
|
||||
dist.all_reduce(counts)
|
||||
assert torch.equal(counts, expected_counts)
|
||||
assert (counts == 512).all()
|
||||
# Each rank keeps the selected keys it owns, as local slots.
|
||||
expected_slots, expected_counts = compact_owned_cache_slots(
|
||||
expected_slots,
|
||||
expected_counts,
|
||||
sharded[2].cache_placement(sharded[0].attn_mqa),
|
||||
)
|
||||
total = counts.clone()
|
||||
dist.all_reduce(total)
|
||||
assert (total == 512).all()
|
||||
else:
|
||||
assert (counts == 512).all()
|
||||
# gfx950 scores KVP candidates with the Triton leaf and one rank with Gluon's.
|
||||
if current_platform().is_amd:
|
||||
return
|
||||
assert torch.equal(counts, expected_counts)
|
||||
for row in range(spec_width):
|
||||
assert sorted(slots[row, :512].tolist()) == sorted(
|
||||
expected_slots[row, :512].tolist()
|
||||
live = int(counts[row])
|
||||
assert sorted(slots[row, :live].tolist()) == sorted(
|
||||
expected_slots[row, :live].tolist()
|
||||
)
|
||||
for got, reference in zip(outputs, expected, strict=True):
|
||||
_assert_close_to_reference(got, reference.float(), fraction)
|
||||
|
||||
@@ -157,11 +157,16 @@ row. That is bit for bit `cuda_dsa_merge_candidates`' selection compacted to
|
||||
the rank's slots, without exchanging positions; `triton_dsa_merge_candidates`
|
||||
selects the same candidates in score order. A score plane reads an invalid
|
||||
candidate off its score, so `scores_out` holds -inf for every -1 offset.
|
||||
Only gfx950 registers it, as one launch over the gathered planes:
|
||||
`dsa_select_owned_candidates_supported` accepts a power-of-two rank count, a
|
||||
power-of-two topk of at least 512, at most 16384 gathered scores per row and
|
||||
rank chunks within 32-bit offsets, and refuses other platforms. A KVP decode
|
||||
whose sizes it refuses merges the candidates instead.
|
||||
gfx950 and SM100/SM103 register it for a topk of 512, 1024, or 2048, each as
|
||||
one launch over the gathered planes. gfx950's leaf accepts a power-of-two rank
|
||||
count, at most 16384 gathered scores per row, and rank chunks within 32-bit
|
||||
offsets. `cute_dsl_dsa_select_owned_candidates` runs one 1024-thread CTA per
|
||||
row: the row's scores stay in registers, three radix digits of 11, 11, and 10
|
||||
bits over a shared-memory histogram find the k-th largest score, and two block
|
||||
scans keep the rank's candidates in candidate order. It accepts two or more
|
||||
power-of-two ranks and at most 32768 gathered scores per row.
|
||||
`dsa_select_owned_candidates_supported` refuses other sizes and platforms; a
|
||||
KVP decode whose sizes it refuses merges the candidates instead.
|
||||
|
||||
## Materialized Triton row top-k
|
||||
|
||||
|
||||
+337
@@ -0,0 +1,337 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Select a KVP rank's share of a merged top-k in one CTA per row.
|
||||
|
||||
The selection reads every rank's gathered candidate scores. Each CTA holds its row's ``ranks * topk`` scores as 32-bit order keys (NaN and
|
||||
-inf as 0, -0 as +0), finds the k-th largest live key for k = ``topk`` (the
|
||||
smallest one with fewer live) in three radix digits of 11, 11, and 10 bits, and
|
||||
keeps this rank's candidates above it, then the ties at it through the ranks in
|
||||
order and within a rank in candidate order, ``topk`` in all: this rank's share
|
||||
of the merge's selection. Kept candidates map to local slots through the local
|
||||
page table in candidate order, ``-1`` after.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cutlass import Int32, Uint32
|
||||
from tokenspeed_kernel.thirdparty.cute_dsl.k3_decode.common.primitives import (
|
||||
float_bits,
|
||||
)
|
||||
from tokenspeed_kernel.thirdparty.cute_dsl.k3_decode.dsa.indexer import (
|
||||
BINS,
|
||||
LOW_BINS,
|
||||
SELECT_THREADS,
|
||||
SELECT_WARPS,
|
||||
block_exclusive,
|
||||
order_key,
|
||||
shared_add,
|
||||
table_entry,
|
||||
topdown,
|
||||
warp_sum,
|
||||
)
|
||||
|
||||
THREADS = SELECT_THREADS
|
||||
# Every rank's scores of a row in registers, at most this many per thread.
|
||||
MAX_KEYS_PER_THREAD = 32
|
||||
NAN_BITS = 0x7F800000
|
||||
NEG_INF_BITS = 0xFF800000
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _live_key(bits: Uint32) -> Uint32:
|
||||
"""Return the order key of FP32 ``bits``, or 0 for NaN and -inf.
|
||||
|
||||
Every live key is above 0, and -0 shares +0's key.
|
||||
"""
|
||||
key = order_key(bits)
|
||||
if (bits & Uint32(0x7FFFFFFF)) > Uint32(NAN_BITS):
|
||||
key = Uint32(0)
|
||||
if bits == Uint32(NEG_INF_BITS):
|
||||
key = Uint32(0)
|
||||
return key
|
||||
|
||||
|
||||
class OwnedSelect:
|
||||
"""Select one rank's share of the merged top-k, one CTA per row.
|
||||
|
||||
Reads ``degree`` ranks' gathered FP32 ``[degree, rows, width]`` scores and
|
||||
writes rank ``rank``'s local slots ``[rows, width]`` and counts ``[rows]``.
|
||||
"""
|
||||
|
||||
def __init__(self, degree: int, width: int, page: int):
|
||||
self.degree, self.width, self.page = degree, width, page
|
||||
self.keys = degree * width // THREADS
|
||||
self.own = max(1, width // THREADS)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
scores: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
table: cute.Tensor,
|
||||
requests: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
lens: cute.Tensor,
|
||||
rank: Int32,
|
||||
stream,
|
||||
):
|
||||
self.kernel(scores, positions, table, requests, out, lens, rank).launch(
|
||||
grid=(out.shape[0], 1, 1), block=(THREADS, 1, 1), stream=stream
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def digit(self, hist, info, keys, shift, mask, match_shift, match, tid):
|
||||
"""Histogram one radix digit of the live keys under a prefix.
|
||||
|
||||
Counts ``(key >> shift) & mask`` over the live keys whose bits above
|
||||
``match_shift`` are ``match`` (every live key at 32).
|
||||
"""
|
||||
cute.arch.sync_threads()
|
||||
for b in cutlass.range(tid, BINS, THREADS, unroll=1):
|
||||
hist[b] = Uint32(0)
|
||||
if tid == 0:
|
||||
info[2] = Int32(0)
|
||||
info[3] = Int32(0)
|
||||
cute.arch.sync_threads()
|
||||
for i in cutlass.range_constexpr(self.keys):
|
||||
key = keys[i]
|
||||
hit = Int32(0)
|
||||
if key != Uint32(0):
|
||||
hit = Int32(1)
|
||||
if cutlass.const_expr(match_shift < 32):
|
||||
if (key >> Uint32(match_shift)) != match:
|
||||
hit = Int32(0)
|
||||
if hit != Int32(0):
|
||||
shared_add(hist.iterator + ((key >> Uint32(shift)) & mask), 1)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
@cute.kernel
|
||||
def kernel(self, scores, positions, table, requests, out, lens, rank):
|
||||
"""Select rank ``rank``'s share of one row's merged top-k.
|
||||
|
||||
Reads the row's scores of every rank, finds the threshold key in three
|
||||
radix digits, then keeps this rank's candidates above it and the ties
|
||||
at it its quota allows (lower ranks' ties first, then candidate
|
||||
order). Writes their local slots through ``positions`` and ``table``
|
||||
to ``out`` in candidate order, ``-1`` after, and the count to ``lens``.
|
||||
"""
|
||||
W = self.width
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
warp = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane = tid % 32
|
||||
|
||||
smem = cutlass.memory.SmemAllocator()
|
||||
hist = smem.allocate_tensor(Uint32, cute.make_layout(BINS), 16)
|
||||
scan = smem.allocate_tensor(Uint32, cute.make_layout(SELECT_WARPS), 16)
|
||||
info = smem.allocate_tensor(Int32, cute.make_layout(8), 16)
|
||||
|
||||
# Column c = tid + THREADS * i of the row: rank c // W, candidate c % W.
|
||||
raw = []
|
||||
for i in cutlass.range_constexpr(self.keys):
|
||||
column = tid + THREADS * i
|
||||
raw.append(scores[column // W, row, column % W])
|
||||
keys = []
|
||||
live = Uint32(0)
|
||||
for i in cutlass.range_constexpr(self.keys):
|
||||
key = _live_key(Uint32(float_bits(raw[i])))
|
||||
if key != Uint32(0):
|
||||
live += Uint32(1)
|
||||
keys.append(key)
|
||||
# This rank's candidates [tid * own, tid * own + own) in candidate
|
||||
# order, with their local slots: loads that overlap the radix digits.
|
||||
first = tid * self.own
|
||||
request = requests[row]
|
||||
own_keys = []
|
||||
slots = []
|
||||
for e in cutlass.range_constexpr(self.own):
|
||||
key = Uint32(0)
|
||||
slot = Int32(-1)
|
||||
if first + e < W:
|
||||
key = _live_key(Uint32(float_bits(scores[rank, row, first + e])))
|
||||
if key != Uint32(0):
|
||||
position = positions[row, first + e]
|
||||
if position >= Int32(0):
|
||||
page = table_entry(table, request, position // Int32(self.page))
|
||||
if page >= Int32(0):
|
||||
slot = page * Int32(self.page) + position % Int32(self.page)
|
||||
own_keys.append(key)
|
||||
slots.append(slot)
|
||||
live = warp_sum(live)
|
||||
if tid == 0:
|
||||
info[4] = Int32(0)
|
||||
cute.arch.sync_threads()
|
||||
if lane == 0:
|
||||
shared_add(info.iterator + 4, live)
|
||||
|
||||
# The threshold: three radix digits over the live keys.
|
||||
self.digit(hist, info, keys, 21, 0x7FF, 32, Uint32(0), tid)
|
||||
need = cutlass.min(Uint32(info[4]), Uint32(W))
|
||||
topdown(hist, BINS, need, info, scan, tid, warp, lane)
|
||||
c0 = Uint32(info[2])
|
||||
need -= Uint32(info[3])
|
||||
self.digit(hist, info, keys, 10, 0x7FF, 21, c0, tid)
|
||||
topdown(hist, BINS, need, info, scan, tid, warp, lane)
|
||||
c1 = Uint32(info[2])
|
||||
need -= Uint32(info[3])
|
||||
prefix = (c0 << Uint32(11)) | c1
|
||||
self.digit(hist, info, keys, 0, 0x3FF, 10, prefix, tid)
|
||||
topdown(hist, LOW_BINS, need, info, scan, tid, warp, lane)
|
||||
threshold = (prefix << Uint32(10)) | Uint32(info[2])
|
||||
# The ties at the threshold to take, through the ranks in order: one
|
||||
# scan sums the lower ranks' ties (high half) and ranks this rank's.
|
||||
take = need - Uint32(info[3])
|
||||
ties = Uint32(0)
|
||||
for i in cutlass.range_constexpr(self.keys):
|
||||
column = tid + THREADS * i
|
||||
if keys[i] == threshold and keys[i] != Uint32(0):
|
||||
if column // W < rank:
|
||||
ties += Uint32(1 << 16)
|
||||
for e in cutlass.range_constexpr(self.own):
|
||||
if own_keys[e] == threshold and own_keys[e] != Uint32(0):
|
||||
ties += Uint32(1)
|
||||
tie_rank, tie_total = block_exclusive(ties, scan, warp, lane)
|
||||
tie_rank = (tie_rank & Uint32(0xFFFF)) + (tie_total >> Uint32(16))
|
||||
kept = Uint32(0)
|
||||
kept_slots = []
|
||||
for e in cutlass.range_constexpr(self.own):
|
||||
key = own_keys[e]
|
||||
slot = slots[e]
|
||||
keep = Int32(0)
|
||||
if key > threshold:
|
||||
keep = Int32(1)
|
||||
if key == threshold and key != Uint32(0):
|
||||
if tie_rank < take:
|
||||
keep = Int32(1)
|
||||
tie_rank += Uint32(1)
|
||||
if keep == Int32(0):
|
||||
slot = Int32(-1)
|
||||
if slot >= Int32(0):
|
||||
kept += Uint32(1)
|
||||
kept_slots.append(slot)
|
||||
at, count = block_exclusive(kept, scan, warp, lane)
|
||||
for e in cutlass.range_constexpr(self.own):
|
||||
if kept_slots[e] >= Int32(0):
|
||||
out[row, Int32(at)] = kept_slots[e]
|
||||
at += Uint32(1)
|
||||
for c in cutlass.range(tid, W, THREADS, unroll=1):
|
||||
if c >= Int32(count):
|
||||
out[row, c] = Int32(-1)
|
||||
if tid == 0:
|
||||
lens[row] = Int32(count)
|
||||
|
||||
|
||||
def _fake(dtype, shape, stride, align):
|
||||
return cute.runtime.make_fake_tensor(
|
||||
dtype, shape, stride=stride, assumed_align=align
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def compile_owned_select(degree: int, width: int, page: int):
|
||||
"""Compile the owned selection for a rank count, width, and page size.
|
||||
|
||||
Selects among ``degree`` ranks' ``width`` candidates per row over
|
||||
``page``-token pages. Rows, the rank stride, the table, and the rank are
|
||||
runtime values: one compile serves every batch.
|
||||
"""
|
||||
sym = cute.sym_int
|
||||
scores = _fake(
|
||||
cutlass.Float32, (degree, sym(), width), (cute.sym_int64(), width, 1), 4
|
||||
)
|
||||
positions = _fake(Int32, (sym(), width), (cute.sym_int64(), 1), 4)
|
||||
table = _fake(Int32, (sym(), sym()), (cute.sym_int64(), 1), 4)
|
||||
requests = _fake(Int32, (sym(),), (1,), 4)
|
||||
out = _fake(Int32, (sym(), width), (width, 1), 4)
|
||||
lens = _fake(Int32, (sym(),), (1,), 4)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
OwnedSelect(degree, width, page),
|
||||
scores,
|
||||
positions,
|
||||
table,
|
||||
requests,
|
||||
out,
|
||||
lens,
|
||||
Int32(0),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def owned_selection_supported(
|
||||
*, ranks: int, rows: int, topk: int, rank_stride: int
|
||||
) -> bool:
|
||||
"""Return whether the owned selection takes these gathered scores.
|
||||
|
||||
The scores are ``ranks`` ranks' ``rows`` rows of ``topk`` candidates,
|
||||
``rank_stride`` elements apart. It takes a power-of-two ``topk`` of at
|
||||
least 512 and a power-of-two rank count whose every-rank row fits the
|
||||
CTA's registers. Host values only.
|
||||
"""
|
||||
keys = ranks * topk
|
||||
return (
|
||||
topk >= 512
|
||||
and topk & (topk - 1) == 0
|
||||
and ranks >= 2
|
||||
and ranks & (ranks - 1) == 0
|
||||
and keys % THREADS == 0
|
||||
and keys // THREADS <= MAX_KEYS_PER_THREAD
|
||||
and rank_stride >= rows * topk
|
||||
)
|
||||
|
||||
|
||||
def cute_dsl_select_owned_from_gathered(
|
||||
scores: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
local_page_table: torch.Tensor,
|
||||
query_requests: torch.Tensor,
|
||||
*,
|
||||
rank: int,
|
||||
topk: int,
|
||||
page_size: int,
|
||||
out: torch.Tensor,
|
||||
lens_out: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Select rank ``rank``'s share of the merged top-k in one launch.
|
||||
|
||||
Reads every rank's gathered FP32 ``[ranks, rows, topk]`` scores (rows
|
||||
contiguous) and writes int32 ``out`` ``[rows, topk]``, the kept local slots
|
||||
first in candidate order and ``-1`` after, and ``lens_out`` ``[rows]``
|
||||
counts.
|
||||
"""
|
||||
degree, rows, width = scores.shape
|
||||
if rows:
|
||||
compile_owned_select(degree, width, page_size)(
|
||||
scores,
|
||||
positions,
|
||||
local_page_table,
|
||||
query_requests,
|
||||
out,
|
||||
lens_out,
|
||||
rank,
|
||||
)
|
||||
return out, lens_out
|
||||
@@ -55,6 +55,7 @@ from tokenspeed_kernel.platform import (
|
||||
pdl_enabled,
|
||||
)
|
||||
from tokenspeed_kernel.registry import Priority, register_kernel
|
||||
from tokenspeed_kernel.selection import traits_problem_filter
|
||||
from tokenspeed_kernel.signature import dense_tensor_format, format_signature
|
||||
|
||||
__all__ = [
|
||||
@@ -783,3 +784,127 @@ if _decode_front is not None:
|
||||
float(softmax_scale),
|
||||
)
|
||||
return q_norm, projected, q_fp8, weights
|
||||
|
||||
|
||||
# Owned selection: a KVP rank's share of the merged top-k in one launch.
|
||||
|
||||
if _fused_indexer is not None:
|
||||
from tokenspeed_kernel.ops.attention.dsa._cute_dsl.owned_selection import (
|
||||
cute_dsl_select_owned_from_gathered,
|
||||
owned_selection_supported,
|
||||
)
|
||||
|
||||
@register_kernel(
|
||||
"attention",
|
||||
"dsa_select_owned_candidates",
|
||||
name="cute_dsl_dsa_select_owned_candidates",
|
||||
solution="cute_dsl",
|
||||
capability=CapabilityRequirement(
|
||||
min_arch_version=ArchVersion(10, 0),
|
||||
max_arch_version=ArchVersion(10, 3),
|
||||
vendors=frozenset({"nvidia"}),
|
||||
),
|
||||
signatures=frozenset(
|
||||
{format_signature(scores=dense_tensor_format(torch.float32))}
|
||||
),
|
||||
traits={
|
||||
"topk": frozenset({512, 1024, 2048}),
|
||||
"problem_filter": frozenset(
|
||||
{
|
||||
traits_problem_filter(
|
||||
owned_selection_supported,
|
||||
"ranks",
|
||||
"rows",
|
||||
"topk",
|
||||
"rank_stride",
|
||||
)
|
||||
}
|
||||
),
|
||||
},
|
||||
priority=Priority.SPECIALIZED,
|
||||
)
|
||||
def cute_dsl_dsa_select_owned_candidates(
|
||||
scores: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
local_page_table: torch.Tensor,
|
||||
query_requests: torch.Tensor,
|
||||
*,
|
||||
rank: int,
|
||||
topk: int,
|
||||
page_size: int,
|
||||
out: torch.Tensor | None = None,
|
||||
lens_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run ``dsa_select_owned_candidates`` in one launch.
|
||||
|
||||
Serves the sizes ``dsa_select_owned_candidates_supported`` admits.
|
||||
|
||||
Args:
|
||||
scores: FP32 ``[ranks, rows, topk]`` every rank's gathered
|
||||
candidate scores (``-inf`` invalid), rows contiguous; the rank
|
||||
stride may be wider.
|
||||
positions: Int32 ``[rows, topk]`` logical positions of this
|
||||
rank's candidates, rows contiguous.
|
||||
local_page_table: Int32 ``[requests, pages]`` this rank's local
|
||||
pages (``-1`` where not owned), rows contiguous.
|
||||
query_requests: Int32 ``[rows]`` request of every row.
|
||||
rank: This rank's index in ``scores``.
|
||||
topk: Entries selected per row over all ranks.
|
||||
page_size: Tokens per page of ``local_page_table``.
|
||||
out: Optional contiguous int32 ``[rows, topk]`` output.
|
||||
lens_out: Optional contiguous int32 ``[rows]`` output.
|
||||
|
||||
Returns:
|
||||
``(out, lens_out)``: this rank's kept candidates as local slots
|
||||
in candidate order, ``-1`` after, and their count per row.
|
||||
"""
|
||||
degree, rows, width = scores.shape
|
||||
if (
|
||||
width != topk
|
||||
or scores.stride()[1:] != (width, 1)
|
||||
or not owned_selection_supported(
|
||||
ranks=degree, rows=rows, topk=topk, rank_stride=scores.stride(0)
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"owned selection needs FP32 [ranks, rows, topk] scores with "
|
||||
"contiguous rows, power-of-two ranks and topk, and every rank's "
|
||||
f"row in registers; got {tuple(scores.shape)} strided "
|
||||
f"{scores.stride()} for top-{topk}"
|
||||
)
|
||||
if not 0 <= rank < degree:
|
||||
raise ValueError(f"rank {rank} is outside {degree} ranks")
|
||||
if any(
|
||||
value.dtype != torch.int32
|
||||
for value in (positions, local_page_table, query_requests)
|
||||
):
|
||||
raise TypeError(
|
||||
"candidate positions, local pages, and requests must be int32"
|
||||
)
|
||||
if positions.shape != (rows, width) or positions.stride(1) != 1:
|
||||
raise ValueError(
|
||||
"this rank's candidate positions need contiguous [rows, topk] rows"
|
||||
)
|
||||
if local_page_table.stride(1) != 1:
|
||||
raise ValueError("local pages need contiguous rows")
|
||||
if query_requests.shape != (rows,) or not query_requests.is_contiguous():
|
||||
raise ValueError("every row needs its request")
|
||||
if out is None:
|
||||
out = torch.empty((rows, topk), device=scores.device, dtype=torch.int32)
|
||||
if lens_out is None:
|
||||
lens_out = torch.empty((rows,), device=scores.device, dtype=torch.int32)
|
||||
if out.shape != (rows, topk) or not out.is_contiguous():
|
||||
raise ValueError("owned slots need a contiguous [rows, topk] output")
|
||||
if lens_out.shape != (rows,) or not lens_out.is_contiguous():
|
||||
raise ValueError("owned counts need a contiguous [rows] output")
|
||||
return cute_dsl_select_owned_from_gathered(
|
||||
scores,
|
||||
positions,
|
||||
local_page_table,
|
||||
query_requests,
|
||||
rank=rank,
|
||||
topk=topk,
|
||||
page_size=page_size,
|
||||
out=out,
|
||||
lens_out=lens_out,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Check the CuTe DSL owned selection against the compacted merge.
|
||||
|
||||
The reference is the merge's selection compacted to each rank's owned slots.
|
||||
Every rank's rows must match bit for bit, with mandatory, signed zero, NaN,
|
||||
short and empty rows and score bands that stress the radix digits and the tie
|
||||
quota; then positions past int32 offsets, one compile across batch rows, and
|
||||
the sizes it refuses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import assert_no_cute_compile, is_nvidia
|
||||
|
||||
if not is_nvidia():
|
||||
pytest.skip("requires an NVIDIA GPU", allow_module_level=True)
|
||||
pytest.importorskip("cutlass")
|
||||
from tokenspeed_kernel import platform # noqa: E402
|
||||
|
||||
if (
|
||||
not platform.ArchVersion(10, 0)
|
||||
<= platform.current_platform().arch_version
|
||||
<= platform.ArchVersion(10, 3)
|
||||
):
|
||||
pytest.skip("requires SM100 or SM103", allow_module_level=True)
|
||||
|
||||
from tokenspeed_kernel.ops.attention.dsa import ( # noqa: E402
|
||||
dsa_select_owned_candidates,
|
||||
dsa_select_owned_candidates_supported,
|
||||
)
|
||||
from tokenspeed_kernel.ops.attention.dsa._triton.candidate_exchange import ( # noqa: E402
|
||||
rank_major_scores,
|
||||
select_through_threshold,
|
||||
)
|
||||
from tokenspeed_kernel.ops.attention.dsa.triton import ( # noqa: E402
|
||||
local_topk_to_global_slots,
|
||||
)
|
||||
from tokenspeed_kernel.ops.kvcache.triton_cache_placement import ( # noqa: E402
|
||||
compact_owned_slots,
|
||||
virtual_slots_to_local,
|
||||
)
|
||||
|
||||
_PAGE, _BLOCK = 64, 128
|
||||
|
||||
|
||||
def _owned_candidates(degree: int, rows: int, q_len: int, topk: int, seed: int):
|
||||
"""Build a virtual index-page table and every rank's owned candidates.
|
||||
|
||||
Returns positions ``[degree, rows, topk]`` (``-1`` invalid) on pages the
|
||||
rank owns and their scores (``-inf`` invalid), with mandatory,
|
||||
signed-zero, NaN, short, and empty rows.
|
||||
"""
|
||||
generator = torch.Generator(device="cuda").manual_seed(seed)
|
||||
requests = rows // q_len
|
||||
blocks_per_request = 24 * degree
|
||||
virtual_blocks = requests * blocks_per_request + 1
|
||||
blocks = 1 + torch.randperm(
|
||||
virtual_blocks - 1, device="cuda", generator=generator
|
||||
).view(requests, blocks_per_request)
|
||||
pages_per_block = _BLOCK // _PAGE
|
||||
table = blocks.unsqueeze(-1) * pages_per_block + torch.arange(
|
||||
pages_per_block, device="cuda"
|
||||
)
|
||||
table = table.view(requests, -1).int()
|
||||
owner = (table // pages_per_block - 1) % degree
|
||||
positions = torch.full((degree, rows, topk), -1, device="cuda", dtype=torch.int32)
|
||||
for source in range(degree):
|
||||
for row in range(rows):
|
||||
pages = (owner[row // q_len] == source).nonzero().flatten()
|
||||
tokens = pages.unsqueeze(1) * _PAGE + torch.arange(_PAGE, device="cuda")
|
||||
tokens = tokens.flatten()
|
||||
pick = torch.randperm(tokens.numel(), device="cuda", generator=generator)
|
||||
pick = tokens[pick[:topk]]
|
||||
positions[source, row, : pick.numel()] = pick.int()
|
||||
scores = torch.rand((degree, rows, topk), device="cuda", generator=generator)
|
||||
scores[:, 0, 3:6] = torch.tensor([torch.inf, torch.inf, -0.0], device="cuda")
|
||||
scores[:, 0, 6] = 0.0
|
||||
scores[:, 0, 7] = torch.nan
|
||||
if rows > 2:
|
||||
# A short history: fewer valid candidates in all than topk.
|
||||
positions[:, 1, topk // (2 * degree) :] = -1
|
||||
# A request step whose history no rank owns.
|
||||
positions[:, 2] = -1
|
||||
scores[positions < 0] = -torch.inf
|
||||
return table, virtual_blocks, positions, scores
|
||||
|
||||
|
||||
def _local_pages(table, virtual_blocks: int, degree: int, rank: int):
|
||||
slots, owned = virtual_slots_to_local(
|
||||
table.long() * _PAGE,
|
||||
rows_per_page=_BLOCK,
|
||||
virtual_block_count=virtual_blocks,
|
||||
degree=degree,
|
||||
rank=rank,
|
||||
)
|
||||
return torch.where(owned, slots // _PAGE, -1).int()
|
||||
|
||||
|
||||
def _merged_then_compacted(table, virtual_blocks, positions, scores, q_len, topk):
|
||||
"""Return every rank's local slots through the full exchange.
|
||||
|
||||
That is the merge's selection (``cuda_dsa_merge_candidates``' threshold
|
||||
selection over an exact top-k), its scheduler slots, then each rank's
|
||||
owned share.
|
||||
"""
|
||||
degree = positions.shape[0]
|
||||
records = torch.stack((positions, scores.view(torch.int32)), dim=-1)
|
||||
merged = rank_major_scores(records)
|
||||
order = torch.topk(merged, topk, dim=-1).indices
|
||||
selected, counts = select_through_threshold(records, merged, order, topk=topk)
|
||||
slots, lens = local_topk_to_global_slots(
|
||||
local_topk_offsets=selected,
|
||||
block_table=table,
|
||||
block_size=_PAGE,
|
||||
q_len_per_req=q_len,
|
||||
)
|
||||
torch.testing.assert_close(lens, counts, atol=0, rtol=0)
|
||||
return [
|
||||
compact_owned_slots(
|
||||
slots,
|
||||
lens,
|
||||
rows_per_page=_BLOCK,
|
||||
virtual_block_count=virtual_blocks,
|
||||
degree=degree,
|
||||
rank=rank,
|
||||
)
|
||||
for rank in range(degree)
|
||||
]
|
||||
|
||||
|
||||
# Eight ranks, 16 requests of 4 verify rows, top-2048: K3.1's KVP8 decode.
|
||||
# The score bands that stress the radix passes and the tie quota run there.
|
||||
@pytest.mark.parametrize(
|
||||
"degree,rows,q_len,topk,band",
|
||||
[(2, 64, 4, 512, "uniform"), (4, 64, 4, 2048, "uniform")]
|
||||
+ [(8, 64, 4, topk, "uniform") for topk in (512, 2048)]
|
||||
+ [(8, 7, 1, 2048, "uniform")]
|
||||
+ [(8, 64, 4, 2048, band) for band in ("tied", "narrow", "single")],
|
||||
)
|
||||
def test_owned_selection_is_each_ranks_share_of_the_merge(
|
||||
degree, rows, q_len, topk, band
|
||||
):
|
||||
table, virtual_blocks, positions, scores = _owned_candidates(
|
||||
degree, rows, q_len, topk, seed=degree * 1000 + rows + topk
|
||||
)
|
||||
if band == "tied":
|
||||
# The cutoff falls inside a tied group that spans the ranks.
|
||||
generator = torch.Generator(device="cuda").manual_seed(rows + topk)
|
||||
ties = torch.randint(
|
||||
0, 8, scores.shape, device="cuda", generator=generator
|
||||
).float()
|
||||
scores = torch.where(positions < 0, scores, ties)
|
||||
elif band == "narrow":
|
||||
# Scores sharing their sign, exponent, and leading mantissa bits,
|
||||
# beside the mandatory, zero, and NaN ones.
|
||||
scores = torch.where(
|
||||
scores.isfinite() & (scores > 0), 5 + scores * 1e-3, scores
|
||||
)
|
||||
elif band == "single":
|
||||
scores = torch.where(positions < 0, scores, 2.5)
|
||||
expected = _merged_then_compacted(
|
||||
table, virtual_blocks, positions, scores, q_len, topk
|
||||
)
|
||||
# Each rank's shard of a send buffer wider than the scores.
|
||||
gathered = torch.full(
|
||||
(degree, rows * topk + 32), torch.nan, device="cuda", dtype=torch.float32
|
||||
)
|
||||
gathered[:, : rows * topk] = scores.view(degree, -1)
|
||||
gathered = gathered[:, : rows * topk].view(degree, rows, topk)
|
||||
requests = (torch.arange(rows, device="cuda") // q_len).int()
|
||||
|
||||
for rank, (slots, counts) in enumerate(expected):
|
||||
local_slots, local_counts = dsa_select_owned_candidates(
|
||||
gathered,
|
||||
positions[rank].contiguous(),
|
||||
_local_pages(table, virtual_blocks, degree, rank),
|
||||
requests,
|
||||
rank=rank,
|
||||
topk=topk,
|
||||
page_size=_PAGE,
|
||||
solution="cute_dsl",
|
||||
)
|
||||
|
||||
torch.testing.assert_close(local_slots, slots, atol=0, rtol=0)
|
||||
torch.testing.assert_close(local_counts, counts, atol=0, rtol=0)
|
||||
total = sum(counts for _, counts in expected)
|
||||
assert (total <= topk).all()
|
||||
assert (total[2] == 0).all()
|
||||
|
||||
|
||||
def test_owned_selection_reads_positions_past_int32_offsets():
|
||||
# This rank's candidate positions in rows an int32 stride apart, the last
|
||||
# row (one with candidates) past 2**31 elements, selected into the
|
||||
# caller's rows of wider buffers.
|
||||
degree, rows, topk = 2, 4, 2048
|
||||
table, virtual_blocks, positions, scores = _owned_candidates(
|
||||
degree, rows, 1, topk, seed=23
|
||||
)
|
||||
(slots, counts), *_ = _merged_then_compacted(
|
||||
table, virtual_blocks, positions, scores, 1, topk
|
||||
)
|
||||
assert int(counts[-1]) > 0
|
||||
stride = (2**31 // (rows - 1) + 64) // 64 * 64
|
||||
storage = torch.empty((rows - 1) * stride + topk, device="cuda", dtype=torch.int32)
|
||||
far = storage.as_strided((rows, topk), (stride, 1))
|
||||
far.copy_(positions[0])
|
||||
assert (rows - 1) * far.stride(0) > 2**31
|
||||
out = torch.full((rows + 2, topk), 7, device="cuda", dtype=torch.int32)
|
||||
lens_out = torch.full((rows + 2,), 7, device="cuda", dtype=torch.int32)
|
||||
|
||||
local_slots, _ = dsa_select_owned_candidates(
|
||||
scores,
|
||||
far,
|
||||
_local_pages(table, virtual_blocks, degree, 0),
|
||||
torch.arange(rows, device="cuda", dtype=torch.int32),
|
||||
rank=0,
|
||||
topk=topk,
|
||||
page_size=_PAGE,
|
||||
solution="cute_dsl",
|
||||
out=out[1 : rows + 1],
|
||||
lens_out=lens_out[1 : rows + 1],
|
||||
)
|
||||
|
||||
assert local_slots.data_ptr() == out[1].data_ptr()
|
||||
torch.testing.assert_close(out[1 : rows + 1], slots, atol=0, rtol=0)
|
||||
torch.testing.assert_close(lens_out[1 : rows + 1], counts, atol=0, rtol=0)
|
||||
assert (out[0] == 7).all() and (out[rows + 1] == 7).all()
|
||||
assert int(lens_out[0]) == 7 and int(lens_out[rows + 1]) == 7
|
||||
|
||||
|
||||
def test_owned_selection_compiles_once_across_batch_rows():
|
||||
def select(rows):
|
||||
table, virtual_blocks, positions, scores = _owned_candidates(
|
||||
8, rows, 1, 2048, seed=rows
|
||||
)
|
||||
dsa_select_owned_candidates(
|
||||
scores,
|
||||
positions[1].contiguous(),
|
||||
_local_pages(table, virtual_blocks, 8, 1),
|
||||
torch.arange(rows, device="cuda", dtype=torch.int32),
|
||||
rank=1,
|
||||
topk=2048,
|
||||
page_size=_PAGE,
|
||||
solution="cute_dsl",
|
||||
)
|
||||
|
||||
select(1)
|
||||
with assert_no_cute_compile():
|
||||
for rows in (3, 8, 37, 64):
|
||||
select(rows)
|
||||
|
||||
|
||||
def test_owned_selection_refuses_gathered_scores_it_cannot_take():
|
||||
assert dsa_select_owned_candidates_supported(
|
||||
ranks=8, rows=64, topk=2048, rank_stride=64 * 2048 + 1152
|
||||
)
|
||||
# Such KVP decodes keep the merge.
|
||||
for ranks, topk, rank_stride in (
|
||||
# Not a power of two.
|
||||
(3, 2048, 2048),
|
||||
# Every rank's share of a row past the registers.
|
||||
(32, 2048, 2048),
|
||||
# A top-k under the CTA's threads' minimum.
|
||||
(2, 256, 256),
|
||||
# Rank chunks that overlap.
|
||||
(8, 2048, 2047),
|
||||
):
|
||||
assert not dsa_select_owned_candidates_supported(
|
||||
ranks=ranks, rows=1, topk=topk, rank_stride=rank_stride
|
||||
)
|
||||
Reference in New Issue
Block a user