mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* fix(kernel): order the batch-memcpy probe against the current stream Adapt the stream dependency and busy-stream regression from #414 (https://github.com/FlashML-org/FreeToken/pull/414). Warm the JIT before queuing the backlog so compilation cannot mask the race. * fix(moe): enable prefill hit-d2d on rocm
76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
from typing import TYPE_CHECKING
|
|
|
|
import torch
|
|
|
|
from .utils import load_jit
|
|
|
|
if TYPE_CHECKING:
|
|
from tvm_ffi import Module
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def _jit_batch_memcpy_module() -> Module:
|
|
return load_jit(
|
|
"batch_memcpy",
|
|
cuda_files=["batch_memcpy.cuh"],
|
|
cuda_wrappers=[("batch_memcpy", "&BatchMemcpy::run")],
|
|
)
|
|
|
|
|
|
def _probe(fn) -> None:
|
|
"""One real 16-byte H2D through the binding. A version-gated build (panic
|
|
branch) or a driver without batch-memcpy support loads cleanly and only fails
|
|
at call time; probing here turns every such mode into a load_batch_memcpy
|
|
exception the caller's fallback path can catch."""
|
|
src = torch.arange(16, dtype=torch.uint8).pin_memory()
|
|
dst = torch.zeros(16, dtype=torch.uint8, device="cuda")
|
|
stream = torch.cuda.Stream()
|
|
# Order the copy after dst's initialization on the current stream;
|
|
# otherwise the fill can overwrite the copied bytes.
|
|
stream.wait_stream(torch.cuda.current_stream())
|
|
fn(
|
|
torch.tensor([dst.data_ptr()]),
|
|
torch.tensor([src.data_ptr()]),
|
|
torch.tensor([16]),
|
|
stream.cuda_stream,
|
|
)
|
|
stream.synchronize()
|
|
if not torch.equal(dst.cpu(), src):
|
|
raise RuntimeError("cudaMemcpyBatchAsync probe copied wrong bytes")
|
|
|
|
|
|
def load_batch_memcpy():
|
|
"""Build (once), probe, and return the batch-memcpy entry point, or raise.
|
|
|
|
The 8-argument cudaMemcpyBatchAsync signature this binding uses is CUDA 13.0's
|
|
(12.8/12.9 had an extra failIdx parameter); gate on the torch runtime version
|
|
before paying for the JIT build, then verify with a real copy.
|
|
HIP uses its own signature, checked by building and probing a real copy.
|
|
"""
|
|
cuda = torch.version.cuda
|
|
if torch.version.hip is None and (
|
|
cuda is None or tuple(int(x) for x in cuda.split(".")[:2]) < (13, 0)
|
|
):
|
|
raise RuntimeError(f"cudaMemcpyBatchAsync binding requires CUDA >= 13.0 (torch built with {cuda})")
|
|
fn = _jit_batch_memcpy_module().batch_memcpy
|
|
_probe(fn)
|
|
return fn
|
|
|
|
|
|
def batch_memcpy_jit(
|
|
dst_ptrs: torch.Tensor,
|
|
src_ptrs: torch.Tensor,
|
|
sizes: torch.Tensor,
|
|
stream: int,
|
|
) -> None:
|
|
"""Enqueue one cudaMemcpyBatchAsync/hipMemcpyBatchAsync of ``len(sizes)`` independent copies.
|
|
|
|
``dst_ptrs``/``src_ptrs``/``sizes`` are same-length CPU int64 tensors of raw
|
|
addresses and byte counts; ``stream`` is a raw cudaStream_t/hipStream_t handle
|
|
(``torch.cuda.Stream.cuda_stream``), which must not be the legacy NULL stream.
|
|
"""
|
|
load_batch_memcpy()(dst_ptrs, src_ptrs, sizes, stream)
|