mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
91 lines
3.5 KiB
Python
91 lines
3.5 KiB
Python
"""Loading progress bars, consistent across the framework's weight-load paths.
|
|
|
|
tqdm-backed (the framework's existing idiom) and byte-oriented -- load time tracks bytes
|
|
moved off disk, not tensor count, so a byte bar shows a meaningful GiB/s. Bars are disabled
|
|
off rank 0 (only the primary should draw) so multi-rank logs stay clean.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from tqdm import tqdm
|
|
|
|
from freetoken.distributed import try_get_tp_info
|
|
|
|
import time
|
|
from typing import Callable, Optional
|
|
|
|
_PROGRESS_SINK: Optional[Callable[[str, int, int], None]] = None
|
|
|
|
|
|
def set_progress_sink(sink: Optional[Callable[[str, int, int], None]]) -> None:
|
|
"""Install (or clear, with None) a global progress callback invoked (throttled)
|
|
by every ``byte_bar`` as ``sink(desc, done_bytes, total_bytes)``. The scheduler's
|
|
rank-0 process installs one that forwards to its ack_queue; call sites are unchanged."""
|
|
global _PROGRESS_SINK
|
|
_PROGRESS_SINK = sink
|
|
|
|
|
|
def _on_primary() -> bool:
|
|
info = try_get_tp_info()
|
|
return info is None or info.is_primary()
|
|
|
|
|
|
def emit_progress(desc: str, done: int, total: int) -> None:
|
|
"""Push a one-off update to the installed sink for a phase that has no ``byte_bar`` — e.g.
|
|
CUDA-graph capture / warmup, which moves no bytes. A ``total <= 0`` reads downstream as an
|
|
indeterminate phase (no percentage). No-op off rank 0 or when no sink is installed."""
|
|
sink = _PROGRESS_SINK
|
|
if sink is not None and _on_primary():
|
|
try:
|
|
sink(desc, done, total)
|
|
except Exception: # noqa: BLE001 — progress reporting must never break load
|
|
pass
|
|
|
|
|
|
class _SinkTqdm(tqdm):
|
|
"""A tqdm that also forwards its progress to the installed ``_PROGRESS_SINK``,
|
|
throttled to <=1 emit / 0.5 s OR a >=1% delta (plus a guaranteed final emit)."""
|
|
|
|
def __init__(self, *args, **kwargs) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self._sink_last_emit = 0.0
|
|
self._sink_last_frac = -1.0
|
|
|
|
def update(self, n: int = 1): # type: ignore[override]
|
|
ret = super().update(n)
|
|
sink = _PROGRESS_SINK
|
|
if sink is not None:
|
|
total = int(self.total or 0)
|
|
done = int(self.n or 0)
|
|
frac = (done / total) if total else 0.0
|
|
now = time.monotonic()
|
|
if (
|
|
now - self._sink_last_emit >= 0.5
|
|
or frac - self._sink_last_frac >= 0.01
|
|
or (total and done >= total)
|
|
):
|
|
self._sink_last_emit = now
|
|
self._sink_last_frac = frac
|
|
try:
|
|
sink(self.desc or "", done, total)
|
|
except Exception: # noqa: BLE001 — progress reporting must never break load
|
|
pass
|
|
return ret
|
|
|
|
|
|
def byte_bar(total: int, desc: str) -> tqdm:
|
|
"""A byte-scaled bar (shows e.g. ``12.8GiB [00:02, 6.1GiB/s]``); ``update(nbytes)`` it
|
|
as each tensor/bank/shard finishes reading. Also drives the progress sink when installed.
|
|
Thread-safe to update from a pool."""
|
|
return _SinkTqdm(total=total, desc=desc, unit="B", unit_scale=True, unit_divisor=1024,
|
|
disable=not _on_primary(), leave=False, dynamic_ncols=True)
|
|
|
|
|
|
def count_bar(iterable, desc: str, total: int | None = None) -> tqdm:
|
|
"""A plain count bar over an iterable (use when total bytes aren't known up front)."""
|
|
return tqdm(iterable, desc=desc, total=total, disable=not _on_primary(),
|
|
leave=False, dynamic_ncols=True)
|
|
|
|
|
|
__all__ = ["byte_bar", "count_bar", "set_progress_sink"]
|