mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat: add simple ROCm GPU support for RDNA3 (gfx1100-1103) - Add hip_compat.h shim mapping CUDA runtime API to HIP equivalents - Update pinned_tensor.cpp to compile under both nvcc and hipcc - Add ROCm detection in arch.py (is_rocm, get_rocm_gfx_arch, is_gfx11xx_family) - Guard NVIDIA arch checks to return None on ROCm - Skip nvcc version check in _toolchain.py when on ROCm - Add ROCm build path in setup.py (ROCM_HOME, amdhip64, --offload-arch) - Add _hip_cflags() in kernel/utils.py for JIT compilation on ROCm - Add is_rocm() and driver_hip_version() in backend.py - Add rocm-smi fallback in __main__.py for clangd generation - Add TODO(ROCm) for NCCL->RCCL, flashinfer/sgl_kernel ROCm builds, Triton autotune RDNA3 tuning, PDL equivalent, hiprtc JIT cache - Add AMD ROCm classifier in pyproject.toml * fix(rocm): fall back for single-bank fast index copy * test(rocm): cover fast index copy fallback * test(rocm): use explicit fast copy module import * fix(rocm): complete RDNA3 runtime path * fix(rocm): preserve CPU MoE implementation * build(rocm): support RDNA4 discovery and modular SDKs * fix(rocm): complete modular SDK runtime support * test(rocm): harden foundation platform contracts * fix(rocm): pin jit target and split install guide * perf(kernel): restore ROCm non-temporal index copy --------- Co-authored-by: bouclem <bouclement2012@gmail.com> Co-authored-by: Yuu <206304251+nekomario28@users.noreply.github.com> Co-authored-by: LZ-QWQ <lz1830837889@163.com>
80 lines
2.0 KiB
Python
80 lines
2.0 KiB
Python
from __future__ import annotations
|
|
|
|
import functools
|
|
from typing import TYPE_CHECKING, Any, Literal
|
|
|
|
from freetoken.env import ENV
|
|
|
|
from .utils import load_aot
|
|
|
|
if TYPE_CHECKING:
|
|
from abc import abstractmethod
|
|
|
|
import torch
|
|
from tvm_ffi import Module
|
|
|
|
class PyNCCLCommunicator:
|
|
@abstractmethod
|
|
def all_reduce(self, input: torch.Tensor, op: Literal["sum"]) -> None: ...
|
|
@abstractmethod
|
|
def all_gather(self, output: torch.Tensor, input: torch.Tensor) -> None: ...
|
|
@abstractmethod
|
|
def get_buffer(self) -> int: ...
|
|
|
|
else:
|
|
PyNCCLCommunicator = Any
|
|
|
|
|
|
@functools.cache
|
|
def _load_nccl_module() -> Module:
|
|
# TODO(ROCm): NCCL -> RCCL migration for multi-GPU tensor parallelism on AMD.
|
|
return load_aot("pynccl", cuda_files=["pynccl.cu"], extra_ldflags=["-lnccl"])
|
|
|
|
|
|
@functools.cache
|
|
def _get_pynccl_wrapper_cls():
|
|
import tvm_ffi
|
|
|
|
@tvm_ffi.register_object("freetoken.NCCLWrapper")
|
|
class PyNCCLImpl(tvm_ffi.Object):
|
|
def __init__(self, *args):
|
|
self.__ffi_init__(*args)
|
|
|
|
return PyNCCLImpl
|
|
|
|
|
|
def init_pynccl(
|
|
*,
|
|
tp_rank: int,
|
|
tp_size: int,
|
|
tp_cpu_group: torch.distributed.ProcessGroup,
|
|
max_size_bytes: int = 0,
|
|
) -> PyNCCLCommunicator:
|
|
import torch
|
|
|
|
max_size_bytes = min(max_size_bytes, ENV.PYNCCL_MAX_BUFFER_SIZE.value)
|
|
|
|
module = _load_nccl_module()
|
|
cls = _get_pynccl_wrapper_cls()
|
|
|
|
if tp_rank == 0:
|
|
id_list = [module.create_nccl_uid()]
|
|
torch.distributed.broadcast_object_list(
|
|
id_list,
|
|
src=0,
|
|
group=tp_cpu_group,
|
|
)
|
|
else:
|
|
id_list = [None]
|
|
torch.distributed.broadcast_object_list(
|
|
id_list,
|
|
src=0,
|
|
group=tp_cpu_group,
|
|
)
|
|
|
|
nccl_id = id_list[0]
|
|
assert not nccl_id is None, f"Failed to get NCCL unique ID on {tp_rank = }"
|
|
|
|
# bypass type checking for the FFI object
|
|
return cls(tp_rank, tp_size, max_size_bytes, nccl_id) # type: ignore
|