Files
FreeToken/python/freetoken/benchmark/perf.py
2026-08-11 22:53:25 +00:00

75 lines
2.0 KiB
Python

from __future__ import annotations
from typing import Any, Callable, Dict, Tuple
from freetoken.utils import init_logger
logger = init_logger(__name__)
def perf_cuda(
f: Callable[[], Any],
*,
init_stream: bool = True,
repetitions: int = 10,
cuda_graph_repetitions: int | None = 10,
) -> float:
import torch
assert repetitions > 0
tic = torch.cuda.Event(enable_timing=True)
toc = torch.cuda.Event(enable_timing=True)
stream = torch.cuda.Stream()
torch.cuda.synchronize()
if init_stream:
stream = torch.cuda.Stream()
else:
stream = torch.cuda.current_stream()
with torch.cuda.stream(stream):
f()
if N := cuda_graph_repetitions:
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
for _ in range(N):
f()
replay = g.replay
del g
else:
replay = f
N = 1
torch.cuda.synchronize()
replay()
tic.record()
for _ in range(repetitions):
replay()
toc.record()
toc.synchronize()
dur = tic.elapsed_time(toc)
return dur / (N * repetitions)
def compare_memory_kernel_perf(
*,
baseline: Callable[[], Any],
our_impl: Callable[[], Any],
memory_footprint: int, # in bytes
description: str = " ",
extra_kwargs: Dict[str, Any] | None = None,
need_latency: bool = True,
) -> Tuple[float, float]:
extra_kwargs = extra_kwargs or {}
dur = perf_cuda(baseline, **extra_kwargs)
bandwidth_0 = memory_footprint / (dur * 1e6) # GB/s
latency_msg = f"{dur:8.3f} ms | " if need_latency else ""
message_0 = f"Baseline: {latency_msg}{bandwidth_0:8.3f} GB/s"
dur = perf_cuda(our_impl, **extra_kwargs)
bandwidth_1 = memory_footprint / (dur * 1e6) # GB/s
latency_msg = f"{dur:8.3f} ms | " if need_latency else ""
logger.info(f"{description}{message_0} | Our Impl: {latency_msg}{bandwidth_1:8.3f} GB/s")
return bandwidth_0, bandwidth_1