mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
75 lines
2.0 KiB
Python
75 lines
2.0 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any, Callable, Dict, Tuple
|
|
|
|
from freetoken.utils import init_logger
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def perf_cuda(
|
|
f: Callable[[], Any],
|
|
*,
|
|
init_stream: bool = True,
|
|
repetitions: int = 10,
|
|
cuda_graph_repetitions: int | None = 10,
|
|
) -> float:
|
|
import torch
|
|
|
|
assert repetitions > 0
|
|
tic = torch.cuda.Event(enable_timing=True)
|
|
toc = torch.cuda.Event(enable_timing=True)
|
|
stream = torch.cuda.Stream()
|
|
torch.cuda.synchronize()
|
|
if init_stream:
|
|
stream = torch.cuda.Stream()
|
|
else:
|
|
stream = torch.cuda.current_stream()
|
|
|
|
with torch.cuda.stream(stream):
|
|
f()
|
|
if N := cuda_graph_repetitions:
|
|
g = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(g):
|
|
for _ in range(N):
|
|
f()
|
|
replay = g.replay
|
|
del g
|
|
else:
|
|
replay = f
|
|
N = 1
|
|
|
|
torch.cuda.synchronize()
|
|
|
|
replay()
|
|
tic.record()
|
|
for _ in range(repetitions):
|
|
replay()
|
|
toc.record()
|
|
toc.synchronize()
|
|
dur = tic.elapsed_time(toc)
|
|
return dur / (N * repetitions)
|
|
|
|
|
|
def compare_memory_kernel_perf(
|
|
*,
|
|
baseline: Callable[[], Any],
|
|
our_impl: Callable[[], Any],
|
|
memory_footprint: int, # in bytes
|
|
description: str = " ",
|
|
extra_kwargs: Dict[str, Any] | None = None,
|
|
need_latency: bool = True,
|
|
) -> Tuple[float, float]:
|
|
extra_kwargs = extra_kwargs or {}
|
|
|
|
dur = perf_cuda(baseline, **extra_kwargs)
|
|
bandwidth_0 = memory_footprint / (dur * 1e6) # GB/s
|
|
latency_msg = f"{dur:8.3f} ms | " if need_latency else ""
|
|
message_0 = f"Baseline: {latency_msg}{bandwidth_0:8.3f} GB/s"
|
|
|
|
dur = perf_cuda(our_impl, **extra_kwargs)
|
|
bandwidth_1 = memory_footprint / (dur * 1e6) # GB/s
|
|
latency_msg = f"{dur:8.3f} ms | " if need_latency else ""
|
|
logger.info(f"{description}{message_0} | Our Impl: {latency_msg}{bandwidth_1:8.3f} GB/s")
|
|
return bandwidth_0, bandwidth_1
|