mirror of
https://github.com/tile-ai/tilelang.git
synced 2026-10-21 06:40:23 +08:00
Lei Wang
c750fb8a98
* [Enhancement] Update examples and tests for improved type handling and functionality
- Enhanced various example scripts to support new data types and improve compatibility with PyTorch.
- Updated tests across multiple modules to ensure correct functionality with the latest changes in type handling.
- Refactored code in examples to streamline operations and improve clarity, particularly in tensor operations and memory management.
- Added comprehensive tests for new features and fixed existing issues related to type conversions and buffer handling.
* [Refactor] Update accumulation data type to float32 across examples
- Changed accumulation data type from "float" to T.float32 in multiple example scripts to ensure consistency and improve numerical stability.
- This update affects various modules including flash attention, GEMM analysis, convolution, and deepseek MLA examples, enhancing type handling across the board.
* [Refactor] Standardize data type usage across benchmark scripts
- Updated data type definitions in benchmark scripts to use T.float16 and T.float32 consistently, enhancing clarity and type handling.
- Adjusted dtype assignments in matmul functions and configuration setups to align with the new standard.
- Improved overall code consistency and maintainability by ensuring uniform data type usage across various modules.
* [Refactor] Standardize data type usage in templates and scripts
- Updated data type definitions in various templates and scripts to use string representations (e.g., "float16", "int32") instead of T.float16 and T.int32 for improved consistency and clarity.
- Enhanced overall code maintainability by ensuring uniform data type usage across multiple modules, including convolution, elementwise operations, and matrix multiplication templates.
- This change aims to streamline type handling and improve compatibility with existing workflows.
* [Refactor] Standardize data type usage in examples and benchmarks
- Updated data type definitions in various example and benchmark scripts to use T.float16 and T.int32 consistently, enhancing clarity and maintainability.
- Adjusted dtype assignments in kernel functions and configuration setups to align with the new standard.
- Improved overall code consistency by ensuring uniform data type usage across multiple modules, including attention mechanisms, matrix multiplication, and GEMM examples.
* [Refactor] Import dtypes from language.v2 module
- Added import statement for dtypes from the language.v2 module to enhance type handling and maintain consistency across the codebase.
- This change aims to streamline data type management and improve overall code clarity.
* fix
* [Refactor] Standardize data type usage across scripts
- Updated data type definitions in various scripts to use string representations (e.g., "float16", "int8") instead of T.float16 and T.int8 for improved consistency and clarity.
- Adjusted dtype assignments in functions and configuration setups to align with the new standard, enhancing overall code maintainability.
- This change affects multiple modules, including benchmark and attention mechanisms, ensuring uniform data type usage throughout the codebase.
* [Refactor] Update data type handling for consistency and clarity
- Changed string representations of data types in the Hint class to use T.float32 and T.int32 for improved consistency.
- Added new data types "int4" and "int16" to the dtypes module, enhancing type support across the codebase.
- Updated function signatures and assertions in the lop3 and mxfp modules to utilize the new data types, ensuring uniformity in type handling.
- This refactor aims to streamline data type management and improve overall code clarity and maintainability.
* [Enhancement] Improve data type handling and error messaging
- Introduced a mapping for canonical data types to their display strings, enhancing clarity in type representation.
- Updated the dtype creation logic to utilize the new mapping, ensuring more intuitive handling of string inputs.
- Refined error messages in the lop3 module to provide clearer feedback on invalid source formats, improving debugging and user experience.
* [Fix] Correct boolean flag in GEMM SP test case
- Updated the boolean flag in the test_gemm_sp_sm90 function to ensure proper functionality in the test case.
- This change enhances the accuracy of the test and aligns it with expected behavior for the GEMM SP implementation.
* [Refactor] Standardize data type usage across scripts
- Updated data type definitions in various scripts to use T.float16 and T.bfloat16 consistently, enhancing clarity and maintainability.
- Adjusted dtype assignments in function signatures and argument parsing to align with the new standard, ensuring uniform data type usage throughout the codebase.
- This change affects multiple modules, including benchmarks and examples, improving overall code consistency and readability.
* [Refactor] Standardize data type usage in various modules
- Updated data type assignments in multiple scripts to utilize T.float32, T.int8, and T.int32 consistently, enhancing clarity and maintainability.
- Adjusted function signatures and parameter types across benchmarks, examples, and tests to align with the new standard, ensuring uniform data type usage throughout the codebase.
- This change improves overall code consistency and readability, impacting modules related to matrix multiplication, GEMM, and tensor operations.
* [Refactor] Update argument parsing for data types in benchmarks
- Changed argument parsing for data types in benchmark_matmul_intrinsic.py and benchmark_matmul_sp.py to use string representations ("float16", "int8", "float") instead of T.float16 and T.float.
- This update enhances consistency in data type handling across benchmark scripts, improving clarity and maintainability.
* [Refactor] Update data type handling in benchmark and example scripts
- Changed data type arguments in benchmark and example scripts to use string representations ("float16") instead of T.float16 for improved consistency.
- Updated function signatures and argument parsing to align with the new standard, enhancing clarity and maintainability across the codebase.
- This change affects multiple modules related to attention mechanisms and tensor operations, ensuring uniform data type usage throughout the examples.
* [Refactor] Fix data type conversion in multiple scripts
- Corrected the usage of the data type conversion method from dtype..as_torch() to dtype.as_torch() across various benchmark and example scripts.
- This change enhances consistency in data type handling and improves code readability, impacting modules related to attention mechanisms and tensor operations.
* [Refactor] Update float8 data type usage across multiple scripts
- Changed instances of T.float8_e4m3 to T.float8_e4m3fn in various benchmark, example, and test scripts to ensure consistency in data type handling.
- This update enhances clarity and maintainability across the codebase, particularly in modules related to matrix multiplication and tensor operations.
* [Refactor] Enhance float8 data type handling in CUDA code generation
- Updated the handling of float8 data types in the CUDA code generation to include additional float8 variants, improving type conversion logic.
- Adjusted conditions to ensure proper type checks for float8 conversions, enhancing clarity and maintainability in the codebase.
- Modified layout inference to streamline float8 type checks, ensuring consistency across the implementation.
- This change impacts modules related to matrix operations and CUDA code generation, improving overall type handling and conversion accuracy.
* [Refactor] Streamline float8 data type handling in CUDA and related modules
- Enhanced float8 data type handling in CUDA code generation by refining type conversion logic and ensuring consistent type checks.
- Updated layout inference for float8 types to improve clarity and maintainability across the implementation.
- This change impacts modules related to matrix operations and CUDA code generation, improving overall type handling and conversion accuracy.
* [Refactor] Remove unnecessary cache disabling in float8 example script
- Eliminated the call to tilelang.disable_cache() in example_group_per_split_token_cast_to_fp8.py to streamline the code.
- This change enhances clarity and maintainability of the example script without affecting its functionality.
* [Refactor] Update data type usage in debug print tests
- Changed the argument for dtype in the test_debug_print_buffer function from a string representation to the corresponding T.bool type.
- This update enhances consistency in data type handling within the test suite, improving clarity and maintainability.
* lint fix
* Update function parameter types from `str` to `T.dtype` for improved type safety in attention sink and related examples
* Refactor `gemv_alloc_reducer` function signature for improved readability by formatting parameters across multiple lines.
540 lines
11 KiB
Python
540 lines
11 KiB
Python
from tilelang import tvm as tvm
|
|
import tilelang.testing
|
|
import tilelang.language as T
|
|
|
|
|
|
def matmul(
|
|
M,
|
|
N,
|
|
K,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
accum_dtype,
|
|
num_stages,
|
|
threads,
|
|
):
|
|
A_shape = (K, M) if trans_A else (M, K)
|
|
B_shape = (N, K) if trans_B else (K, N)
|
|
A_shared_shape = (block_K, block_M) if trans_A else (block_M, block_K)
|
|
B_shared_shape = (block_N, block_K) if trans_B else (block_K, block_N)
|
|
|
|
@T.prim_func
|
|
def main(
|
|
A: T.Tensor(A_shape, in_dtype),
|
|
B: T.Tensor(B_shape, in_dtype),
|
|
C: T.Tensor((M, N), out_dtype),
|
|
):
|
|
with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=threads) as (bx, by):
|
|
A_shared = T.alloc_shared(A_shared_shape, in_dtype)
|
|
B_shared = T.alloc_shared(B_shared_shape, in_dtype)
|
|
C_local = T.alloc_fragment((block_M, block_N), accum_dtype)
|
|
T.clear(C_local)
|
|
for k in T.Pipelined(T.ceildiv(K, block_K), num_stages=num_stages):
|
|
if trans_A:
|
|
T.copy(A[k * block_K, by * block_M], A_shared)
|
|
else:
|
|
T.copy(A[by * block_M, k * block_K], A_shared)
|
|
if trans_B:
|
|
T.copy(B[bx * block_N, k * block_K], B_shared)
|
|
else:
|
|
T.copy(B[k * block_K, bx * block_N], B_shared)
|
|
T.gemm(A_shared, B_shared, C_local, trans_A, trans_B)
|
|
T.copy(C_local, C[by * block_M, bx * block_N])
|
|
|
|
return main
|
|
|
|
|
|
def run_gemm(
|
|
M,
|
|
N,
|
|
K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
dtypeAccum,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
num_stages=0,
|
|
num_threads=128,
|
|
):
|
|
program = matmul(
|
|
M,
|
|
N,
|
|
K,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
dtypeAccum,
|
|
num_stages,
|
|
num_threads,
|
|
)
|
|
|
|
kernel = tilelang.compile(program, out_idx=[2])
|
|
print(kernel.get_kernel_source())
|
|
profiler = kernel.get_profiler()
|
|
|
|
def ref_program(A, B):
|
|
import torch
|
|
|
|
if trans_A:
|
|
A = A.T
|
|
if trans_B:
|
|
B = B.T
|
|
if in_dtype == T.float32:
|
|
# Convert float32 to tfloat32 because tfloat32 mma cannot truncate
|
|
# float32 automatically, -0x1000 meas
|
|
A = (A.view(torch.int32) - 0x1000).view(torch.float32)
|
|
B = (B.view(torch.int32) - 0x1000).view(torch.float32)
|
|
C = torch.matmul(A.to(torch.float), B.to(torch.float))
|
|
C = C.to(torch.__getattribute__(out_dtype))
|
|
return C
|
|
|
|
profiler.assert_allclose(ref_program, atol=1e-2, rtol=1e-2)
|
|
|
|
|
|
def test_gemm_f16f16f16_nn():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
128,
|
|
32,
|
|
0,
|
|
)
|
|
|
|
|
|
def test_gemm_f16f16f32_nn():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float32,
|
|
128,
|
|
128,
|
|
32,
|
|
)
|
|
|
|
|
|
def test_gemm_bf16bf16f32_nn():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
False,
|
|
T.bfloat16,
|
|
T.bfloat16,
|
|
T.float32,
|
|
128,
|
|
128,
|
|
32,
|
|
)
|
|
|
|
|
|
def test_gemm_f32f32f32_nn():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
False,
|
|
T.float32,
|
|
T.float32,
|
|
T.float32,
|
|
64,
|
|
128,
|
|
32,
|
|
)
|
|
|
|
|
|
def test_gemm_f16f16f16_tn():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
True,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
128,
|
|
32,
|
|
0,
|
|
)
|
|
|
|
|
|
def test_gemm_f16f16f16_nt():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
True,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
128,
|
|
32,
|
|
0,
|
|
)
|
|
|
|
|
|
def test_gemm_i8i8i32_nt():
|
|
run_gemm(512, 1024, 768, False, True, T.int8, T.int8, T.int32, 128, 128, 64)
|
|
|
|
|
|
def test_gemm_i8i8i32_tn():
|
|
run_gemm(512, 1024, 768, True, False, T.int8, T.int8, T.int32, 128, 128, 64)
|
|
|
|
|
|
def test_gemm_f64f64f64_nt():
|
|
run_gemm(512, 512, 512, False, True, T.float64, T.float64, T.float64, 64, 32, 16)
|
|
|
|
|
|
def test_gemm_f32f32f32_nt():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
True,
|
|
T.float32,
|
|
T.float32,
|
|
T.float32,
|
|
64,
|
|
128,
|
|
32,
|
|
)
|
|
|
|
|
|
def test_gemm_f32f32f32_tn():
|
|
run_gemm(
|
|
512,
|
|
1024,
|
|
768,
|
|
True,
|
|
False,
|
|
T.float32,
|
|
T.float32,
|
|
T.float32,
|
|
64,
|
|
128,
|
|
32,
|
|
)
|
|
|
|
|
|
def test_pad_aligned_f16f16f16_nn():
|
|
run_gemm(
|
|
512 - 8,
|
|
1024 - 32,
|
|
768 - 24,
|
|
False,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
256,
|
|
32,
|
|
2,
|
|
)
|
|
|
|
|
|
def test_pad_f16f16f16_nn():
|
|
run_gemm(
|
|
512 - 9,
|
|
1024 - 7,
|
|
768 - 5,
|
|
False,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
256,
|
|
32,
|
|
2,
|
|
)
|
|
|
|
|
|
def test_pad_f16f16f32_nn():
|
|
run_gemm(
|
|
512 + 19,
|
|
1024 + 17,
|
|
768 + 15,
|
|
False,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float32,
|
|
128,
|
|
64,
|
|
32,
|
|
)
|
|
|
|
|
|
def matmul_sr(
|
|
M,
|
|
N,
|
|
K,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
accum_dtype,
|
|
num_stages,
|
|
threads,
|
|
):
|
|
A_shape = (K, M) if trans_A else (M, K)
|
|
B_shape = (N, K) if trans_B else (K, N)
|
|
A_shared_shape = (block_K, block_M) if trans_A else (block_M, block_K)
|
|
B_shared_shape = (block_N, block_K) if trans_B else (block_K, block_N)
|
|
|
|
import tilelang.language as T
|
|
|
|
@T.prim_func
|
|
def main(
|
|
A: T.Tensor(A_shape, in_dtype),
|
|
B: T.Tensor(B_shape, in_dtype),
|
|
C: T.Tensor((M, N), out_dtype),
|
|
):
|
|
with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=threads) as (bx, by):
|
|
A_shared = T.alloc_shared(A_shared_shape, in_dtype)
|
|
B_shared = T.alloc_shared(B_shared_shape, in_dtype)
|
|
B_local = T.alloc_fragment(B_shared_shape, in_dtype)
|
|
C_local = T.alloc_fragment((block_M, block_N), accum_dtype)
|
|
T.clear(C_local)
|
|
for k in T.Pipelined(T.ceildiv(K, block_K), num_stages=num_stages):
|
|
if trans_A:
|
|
T.copy(A[k * block_K, by * block_M], A_shared)
|
|
else:
|
|
T.copy(A[by * block_M, k * block_K], A_shared)
|
|
if trans_B:
|
|
T.copy(B[bx * block_N, k * block_K], B_shared)
|
|
T.copy(B_shared, B_local)
|
|
else:
|
|
T.copy(B[k * block_K, bx * block_N], B_shared)
|
|
T.copy(B_shared, B_local)
|
|
T.gemm(A_shared, B_local, C_local, trans_A, trans_B)
|
|
T.copy(C_local, C[by * block_M, bx * block_N])
|
|
|
|
return main
|
|
|
|
|
|
def run_gemm_sr(
|
|
M,
|
|
N,
|
|
K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
dtypeAccum,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
num_stages=1,
|
|
num_threads=128,
|
|
):
|
|
program = matmul_sr(
|
|
M,
|
|
N,
|
|
K,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
dtypeAccum,
|
|
num_stages,
|
|
num_threads,
|
|
)
|
|
|
|
kernel = tilelang.compile(program, out_idx=[2])
|
|
profiler = kernel.get_profiler()
|
|
|
|
def ref_program(A, B):
|
|
import torch
|
|
|
|
if trans_A:
|
|
A = A.T
|
|
if trans_B:
|
|
B = B.T
|
|
A = A.to(torch.float)
|
|
B = B.to(torch.float)
|
|
C = torch.matmul(A, B)
|
|
C = C.to(torch.__getattribute__(out_dtype))
|
|
return C
|
|
|
|
profiler.assert_allclose(ref_program, atol=1e-2, rtol=1e-2)
|
|
|
|
|
|
# WGMMA only supports B in shared
|
|
@tilelang.testing.requires_cuda_compute_version_le(8, 9)
|
|
def test_gemm_f16f16f16_sr():
|
|
run_gemm_sr(
|
|
512,
|
|
1024,
|
|
768,
|
|
False,
|
|
True,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
128,
|
|
32,
|
|
0,
|
|
)
|
|
|
|
|
|
def matmul_rs(
|
|
M,
|
|
N,
|
|
K,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
accum_dtype,
|
|
num_stages,
|
|
threads,
|
|
):
|
|
A_shape = (K, M) if trans_A else (M, K)
|
|
B_shape = (N, K) if trans_B else (K, N)
|
|
A_shared_shape = (block_K, block_M) if trans_A else (block_M, block_K)
|
|
B_shared_shape = (block_N, block_K) if trans_B else (block_K, block_N)
|
|
|
|
import tilelang.language as T
|
|
|
|
@T.prim_func
|
|
def main(
|
|
A: T.Tensor(A_shape, in_dtype),
|
|
B: T.Tensor(B_shape, in_dtype),
|
|
C: T.Tensor((M, N), out_dtype),
|
|
):
|
|
with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=threads) as (bx, by):
|
|
A_shared = T.alloc_shared(A_shared_shape, in_dtype, scope="shared")
|
|
A_local = T.alloc_fragment(A_shared_shape, in_dtype)
|
|
B_shared = T.alloc_shared(B_shared_shape, in_dtype, scope="shared")
|
|
C_local = T.alloc_fragment((block_M, block_N), accum_dtype)
|
|
T.clear(C_local)
|
|
for k in T.Pipelined(T.ceildiv(K, block_K), num_stages=num_stages):
|
|
if trans_A:
|
|
T.copy(A[k * block_K, by * block_M], A_shared)
|
|
T.copy(A_shared, A_local)
|
|
else:
|
|
T.copy(A[by * block_M, k * block_K], A_shared)
|
|
T.copy(A_shared, A_local)
|
|
if trans_B:
|
|
T.copy(B[bx * block_N, k * block_K], B_shared)
|
|
else:
|
|
T.copy(B[k * block_K, bx * block_N], B_shared)
|
|
T.gemm(A_local, B_shared, C_local, trans_A, trans_B)
|
|
T.copy(C_local, C[by * block_M, bx * block_N])
|
|
|
|
return main
|
|
|
|
|
|
def run_gemm_rs(
|
|
M,
|
|
N,
|
|
K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
dtypeAccum,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
num_stages=1,
|
|
num_threads=128,
|
|
):
|
|
program = matmul_rs(
|
|
M,
|
|
N,
|
|
K,
|
|
block_M,
|
|
block_N,
|
|
block_K,
|
|
trans_A,
|
|
trans_B,
|
|
in_dtype,
|
|
out_dtype,
|
|
dtypeAccum,
|
|
num_stages,
|
|
num_threads,
|
|
)
|
|
|
|
kernel = tilelang.compile(program, out_idx=[2])
|
|
print(kernel.get_kernel_source())
|
|
profiler = kernel.get_profiler()
|
|
|
|
def ref_program(A, B):
|
|
import torch
|
|
|
|
if trans_A:
|
|
A = A.T
|
|
if trans_B:
|
|
B = B.T
|
|
C = torch.matmul(A.to(torch.float), B.to(torch.float))
|
|
C = C.to(torch.__getattribute__(out_dtype))
|
|
return C
|
|
|
|
profiler.assert_allclose(ref_program, atol=1e-2, rtol=1e-2)
|
|
|
|
|
|
# Register source A operand GMMAs must have K-major A layout.
|
|
@tilelang.testing.requires_cuda_compute_version_le(8, 9)
|
|
def test_gemm_f16f16f16_rs():
|
|
run_gemm_rs(
|
|
512,
|
|
1024,
|
|
768,
|
|
True,
|
|
False,
|
|
T.float16,
|
|
T.float16,
|
|
T.float16,
|
|
128,
|
|
128,
|
|
32,
|
|
0,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
tilelang.testing.main()
|