tilelang/testing/python/issue/test_tilelang_issue_2928.py

"""Regression test for GitHub issue #2928.

A ``block_N`` that is itself a legal Hopper WGMMA width (96, 112, 160, ...)
must lower to a single ``m64nNk16`` instruction.  Deriving the instruction
width from ``gcd(warp_col_tiles, 256)`` decomposed those extents into n32 / n16
atoms and gave up most of the tensor-core throughput.
"""

import re

import pytest
import torch

import tilelang
import tilelang.language as T
import tilelang.testing
from tilelang.cuda.intrinsics.macro.wgmma_macro_generator import select_wgmma_inst_n


@pytest.mark.parametrize(
    "warp_col_tiles, expected",
    [
        # Legal single-instruction extents: keep the whole width.
        (64, 64),
        (96, 96),
        (112, 112),
        (160, 160),
        (176, 176),
        (256, 256),
        # ``N % 16 == 8`` is a separately broken class (issue #2593); the width
        # selection is left exactly as it was.
        (24, 8),
        (40, 8),
        (88, 8),
        # Past the 256 ceiling: widest legal divisor, which gcd cannot reach.
        (384, 192),
        (512, 256),
    ],
)
def test_select_wgmma_inst_n(warp_col_tiles, expected):
    assert select_wgmma_inst_n(warp_col_tiles) == expected


def _matmul_nt(M, N, K, block_M, block_N, block_K):
    dtype, accum_dtype = "float16", "float"

    @T.prim_func
    def main(
        A: T.Tensor((M, K), dtype),
        B: T.Tensor((N, K), dtype),
        C: T.Tensor((M, N), dtype),
    ):
        with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=128) as (bx, by):
            A_shared = T.alloc_shared((block_M, block_K), dtype)
            B_shared = T.alloc_shared((block_N, block_K), dtype)
            C_local = T.alloc_fragment((block_M, block_N), accum_dtype)
            T.clear(C_local)
            for k in T.Pipelined(T.ceildiv(K, block_K), num_stages=2):
                T.copy(A[by * block_M, k * block_K], A_shared)
                T.copy(B[bx * block_N, k * block_K], B_shared)
                T.gemm(A_shared, B_shared, C_local, transpose_B=True)
            T.copy(C_local, C[by * block_M, bx * block_N])

    return main


def _emitted_shapes(kernel, op):
    """Distinct ``m, n, k`` operands emitted for ``op`` (``wgmma_ss``, ``wgmma_sp_ss``)."""
    return sorted(set(re.findall(rf"{op}<[^>]*?, (\d+, \d+, \d+),", kernel.get_kernel_source())))


@tilelang.testing.requires_cuda_compute_version_eq(9, 0)
@pytest.mark.parametrize("block_N", [96, 112, 160])
def test_gemm_emits_single_wide_wgmma(block_N):
    M, K, block_M, block_K = 128, 128, 64, 64
    N = block_N * 2

    kernel = tilelang.compile(_matmul_nt(M, N, K, block_M, block_N, block_K))
    assert _emitted_shapes(kernel, "wgmma_ss") == [f"64, {block_N}, 16"]

    torch.manual_seed(0)
    a = torch.randn(M, K, dtype=torch.float16, device="cuda")
    b = torch.randn(N, K, dtype=torch.float16, device="cuda")
    c = torch.empty(M, N, dtype=torch.float16, device="cuda")
    kernel(a, b, c)
    torch.testing.assert_close(c, (a.float() @ b.float().T).half(), rtol=1e-2, atol=1e-2)


def _matmul_sp_nt(M, N, K, block_M, block_N, block_K, e_factor):
    dtype, accum_dtype, meta_dtype = "float16", "float", "int16"

    @T.prim_func
    def main(
        A_sparse: T.Tensor((M, K // 2), dtype),
        E: T.Tensor((M, K // e_factor), meta_dtype),
        B: T.Tensor((N, K), dtype),
        C: T.Tensor((M, N), dtype),
    ):
        with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=128) as (bx, by):
            A_shared = T.alloc_shared((block_M, block_K // 2), dtype)
            E_shared = T.alloc_shared((block_M, block_K // e_factor), meta_dtype)
            B_shared = T.alloc_shared((block_N, block_K), dtype)
            C_frag = T.alloc_fragment((block_M, block_N), accum_dtype)
            T.clear(C_frag)
            for k in T.Pipelined(T.ceildiv(K, block_K), num_stages=2):
                T.copy(E[by * block_M, k * block_K // e_factor], E_shared)
                T.copy(A_sparse[by * block_M, k * block_K // 2], A_shared)
                T.copy(B[bx * block_N, k * block_K], B_shared)
                T.gemm_sp(A_shared, E_shared, B_shared, C_frag, False, False, True)
            T.copy(C_frag, C[by * block_M, bx * block_N])

    return main


@tilelang.testing.requires_cuda_compute_version_eq(9, 0)
@pytest.mark.parametrize("block_N", [96, 112, 160])
def test_gemm_sp_emits_single_wide_wgmma(block_N):
    # Imported inside the sm_90 gate so collection never depends on examples/.
    from examples.gemm_sp.sparse_utils import compress, get_e_factor, randn_semi_sparse

    M, K, block_M, block_K = 128, 32, 128, 32
    N = block_N * 2
    e_factor = get_e_factor(T.float16, T.int16)

    kernel = tilelang.compile(
        _matmul_sp_nt(M, N, K, block_M, block_N, block_K, e_factor),
        out_idx=[3],
        pass_configs={tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True},
    )
    # Sparse WGMMA doubles K per instruction: 512 bits / 16 bits per element.
    assert _emitted_shapes(kernel, "wgmma_sp_ss") == [f"64, {block_N}, 32"]

    torch.manual_seed(0)
    a = randn_semi_sparse(M, K, dtype=torch.float16, device="cuda")
    b = torch.randn(N, K, dtype=torch.float16, device="cuda")
    a_sparse, e = compress(a, meta_dtype=torch.int16)
    c = kernel(a_sparse, e, b)
    torch.testing.assert_close(c.float(), a.float() @ b.float().T, rtol=1e-2, atol=1e-2)


if __name__ == "__main__":
    tilelang.testing.main()