mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat: add --embed-weights host for a pinned host embedding table * refactor: check the tie from the model config and share the op walk
55 lines
1.5 KiB
Python
55 lines
1.5 KiB
Python
from __future__ import annotations
|
|
|
|
import functools
|
|
from typing import TYPE_CHECKING, Tuple
|
|
|
|
from .utils import KernelConfig, load_jit, make_cpp_args
|
|
|
|
if TYPE_CHECKING:
|
|
import torch
|
|
from tvm_ffi import Module
|
|
|
|
DEFAULT_INDEX_KERNEL_CONFIG = KernelConfig(num_threads=128, max_occupancy=1, use_pdl=False)
|
|
|
|
|
|
@functools.cache
|
|
def _jit_index_module(
|
|
element_size: int,
|
|
*,
|
|
num_splits: int = 1,
|
|
config: KernelConfig = DEFAULT_INDEX_KERNEL_CONFIG,
|
|
) -> Module:
|
|
args = make_cpp_args(element_size, num_splits, *config)
|
|
return load_jit(
|
|
"index",
|
|
*args,
|
|
cuda_files=["index.cu"],
|
|
cuda_wrappers=[("launch", f"IndexKernel<{args}>::run")],
|
|
)
|
|
|
|
|
|
def num_splits_for(element_size: int) -> int:
|
|
"""Split factor for a row of ``element_size`` bytes; also used by the AOT
|
|
shape table (kernel/aot_models.py), which must reproduce it exactly."""
|
|
if element_size % 2048 == 0:
|
|
return 4
|
|
if element_size % 1024 == 0:
|
|
return 2
|
|
return 1
|
|
|
|
|
|
def indexing(
|
|
weights: torch.Tensor,
|
|
indices: torch.Tensor,
|
|
*,
|
|
output: torch.Tensor | None = None,
|
|
vocab_range: Tuple[int, int] | None = None, # (start, length)
|
|
) -> torch.Tensor:
|
|
if output is None:
|
|
output = weights.new_empty(indices.shape[0], weights.shape[1], device=indices.device)
|
|
|
|
element_size = weights.shape[1] * weights.element_size()
|
|
module = _jit_index_module(element_size, num_splits=num_splits_for(element_size))
|
|
module.launch(weights, indices, output, vocab_range)
|
|
return output
|