import pytest import tilelang import tilelang.language as T import torch from tvm import tirx import tilelang.testing @tilelang.jit def kernel_with_warp_sync(): @T.prim_func def main( A: T.Tensor((1,), "int32"), B: T.Tensor((1,), "int32"), ): with T.Kernel(1, threads=32): tx = T.get_thread_binding() if tx == 0: tirx.call_extern("void", "__nanosleep", 100) A[0] = -1 T.sync_warp() if tx == 1: B[0] = A[0] return main @tilelang.testing.requires_cuda def test_warp_sync(): a = torch.empty((1), device="cuda", dtype=torch.int32) b = torch.empty((1), device="cuda", dtype=torch.int32) kernel = kernel_with_warp_sync() assert "__syncwarp" in kernel.get_kernel_source() kernel(a, b) assert b[0] == -1 @tilelang.jit def kernel_with_shfl_sync(): @T.prim_func def main( A: T.Tensor((32,), "int32"), ): with T.Kernel(1, threads=32): tx = T.get_thread_binding() val = tx * 10 broadcast = T.shfl_sync(val, 31) A[tx] = broadcast return main @tilelang.testing.requires_cuda def test_shfl_sync(): a = torch.empty((32), device="cuda", dtype=torch.int32) kernel = kernel_with_shfl_sync() assert "__shfl_sync" in kernel.get_kernel_source() kernel(a) assert torch.all(a == 310) def _shift(lane, delta): """Return the source lane, or the current lane when the shift is out of range.""" shifted = lane + delta return shifted if 0 <= shifted < 32 else lane # Builtin each shuffle lowers to and the lane its result comes from, in the row # order shfl_all_ops_kernel writes. The tables differ so the two shapes cannot fold. SHFL_TEMP_OPS = ( ("__shfl_sync", lambda lane: 31), ("__shfl_xor_sync", lambda lane: lane ^ 1), ("__shfl_down_sync", lambda lane: _shift(lane, 1)), ("__shfl_up_sync", lambda lane: _shift(lane, -1)), ) SHFL_INLINE_OPS = ( ("__shfl_sync", lambda lane: 0), ("__shfl_xor_sync", lambda lane: lane ^ 2), ("__shfl_down_sync", lambda lane: _shift(lane, 2)), ("__shfl_up_sync", lambda lane: _shift(lane, -2)), ) # One distinct value per lane so a wrong source lane cannot match by accident, on # float8_e5m2's coarse grid. Lane 0's negative zero needs the byte comparison. _MANTISSAS = (1.0, 1.25, 1.5, 1.75) _GRID = [sign * mant * 2.0**exp for exp in range(4) for mant in _MANTISSAS for sign in (1, -1)] LANE_VALUES = [-0.0, *_GRID[1:]] def shfl_all_ops_kernel(dtype): @T.prim_func def main( A: T.Tensor((32,), dtype), B: T.Tensor((len(SHFL_TEMP_OPS) + len(SHFL_INLINE_OPS), 32), dtype), ): with T.Kernel(1, threads=32): tx = T.get_thread_binding() # Rows 0-3 copy-initialize the result, rows 4-7 store it inline. broadcast = T.shfl_sync(A[tx], 31) swapped = T.shfl_xor(A[tx], 1) shifted_down = T.shfl_down(A[tx], 1) shifted_up = T.shfl_up(A[tx], 1) B[0, tx] = broadcast B[1, tx] = swapped B[2, tx] = shifted_down B[3, tx] = shifted_up B[4, tx] = T.shfl_sync(A[tx], 0) B[5, tx] = T.shfl_xor(A[tx], 2) B[6, tx] = T.shfl_down(A[tx], 2) B[7, tx] = T.shfl_up(A[tx], 2) return main @tilelang.testing.requires_cuda @pytest.mark.parametrize("dtype", [T.float16, T.bfloat16, T.float8_e4m3, T.float8_e5m2]) def test_shfl_narrow_float_dtypes(dtype): kernel = tilelang.compile(shfl_all_ops_kernel(dtype)) source = kernel.get_kernel_source() values = torch.tensor(LANE_VALUES, device="cuda", dtype=torch.float32) torch_dtype = dtype.as_torch() rows = len(SHFL_TEMP_OPS) + len(SHFL_INLINE_OPS) # out_idx would rebuild a torch tensor from float8 DLPack, unsupported on some versions. b = torch.empty(rows, 32, device="cuda", dtype=torch_dtype) kernel(values.to(torch_dtype), b) # Both shapes must still reach the raw builtin, or these overloads stop being # covered. Count in the kernel body, not the preamble. body = source[source.rindex("__global__") :] for builtin, _ in SHFL_TEMP_OPS: assert body.count(f"{builtin}(") == 2 for base, ops, form in ((0, SHFL_TEMP_OPS, "temporary"), (len(SHFL_TEMP_OPS), SHFL_INLINE_OPS, "inline")): for offset, (builtin, src_lane) in enumerate(ops): ref = values[[src_lane(lane) for lane in range(32)]].to(torch_dtype) # Compare bytes: a float comparison would not separate -0.0 from +0.0. got_bytes = b[base + offset].view(torch.uint8) ref_bytes = ref.view(torch.uint8) assert torch.equal(got_bytes, ref_bytes), ( f"{builtin} ({form} form) shuffled the wrong lane: got {got_bytes.tolist()}, expected {ref_bytes.tolist()}" ) if __name__ == "__main__": tilelang.testing.main()