mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
feat(engine): report resident tokens in the load snapshot
The load snapshot gains num_resident_tokens: the prompt plus committed output tokens of every request the rank holds, summed over the output processor's request states each time the loop samples load. Page counts do not measure a rank's attention work (hybrid packing and state reservations make them drift from the tokens attended), so a gateway placing decode requests by context needs the token count itself. The smg Python servicer forwards it as SchedulerLoad.num_resident_tokens. The direct ZMQ lane does not carry it. Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bc58c927ad
commit
94b30e81ac
@@ -425,6 +425,10 @@ Cached-only parents are `num_usable_pages - empty - active`. Active and cached
|
||||
counts are disjoint, even when cache groups pack multiple blocks into a parent.
|
||||
`LoadSnapshot.num_used_pages` sums active and cached parents to report all
|
||||
resident occupancy, including evictable cache, rather than cache alone.
|
||||
Parents do not measure a rank's attention work: hybrid packing and state
|
||||
reservations make their count drift from the tokens attended. The snapshot
|
||||
therefore also carries `num_resident_tokens`, the prompt plus committed
|
||||
output of every request the rank holds, read from the output processor.
|
||||
`available_lcm_blocks()` scans the pool and prefix indexes for reclaimability
|
||||
and belongs in diagnostics and leak checks, never in the per-round sampler.
|
||||
|
||||
|
||||
@@ -1131,6 +1131,12 @@ class EventLoop:
|
||||
def _num_running(self) -> int:
|
||||
return len(self.output_processor.rid_to_state)
|
||||
|
||||
def _num_resident_tokens(self) -> int:
|
||||
return sum(
|
||||
state.input_length + state.output_length
|
||||
for state in self.output_processor.rid_to_state.values()
|
||||
)
|
||||
|
||||
def _request_context_length(self, rid: str) -> int:
|
||||
"""Tokens ``rid`` holds right now: its prompt plus everything committed."""
|
||||
state = self.output_processor.rid_to_state[rid]
|
||||
@@ -1301,7 +1307,9 @@ class EventLoop:
|
||||
|
||||
forward_op = self._get_forward_op(execution_plan)
|
||||
stats = self._get_scheduler_stats()
|
||||
self.load_reporter.observe(stats, self._num_running())
|
||||
self.load_reporter.observe(
|
||||
stats, self._num_running(), self._num_resident_tokens()
|
||||
)
|
||||
num_iter_tokens = (
|
||||
sum(forward_op.input_lengths) if forward_op is not None else 0
|
||||
)
|
||||
@@ -1431,7 +1439,9 @@ class EventLoop:
|
||||
if self._pause.forward_blocked:
|
||||
# Frozen rounds take no planning sample of their own; the
|
||||
# idle sleep bounds this to one sample per millisecond.
|
||||
self.load_reporter.sample_and_observe(self._num_running())
|
||||
self.load_reporter.sample_and_observe(
|
||||
self._num_running(), self._num_resident_tokens()
|
||||
)
|
||||
|
||||
# Resolve a deferred abort/wait pause reply once in-flight work drains.
|
||||
self._pause.maybe_finish_drain(self.scheduler)
|
||||
|
||||
@@ -1191,6 +1191,8 @@ class LoadSnapshot(
|
||||
num_used_pages: int
|
||||
max_total_pages: int
|
||||
valid_for_ms: int
|
||||
# Prompt plus committed output tokens summed over the rank's requests.
|
||||
num_resident_tokens: int
|
||||
|
||||
|
||||
class BlockReqType(Enum):
|
||||
|
||||
@@ -39,7 +39,7 @@ from tokenspeed.runtime.engine.io_struct import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LoadValues = tuple[int, int, int, int, int]
|
||||
_LoadValues = tuple[int, int, int, int, int, int]
|
||||
_SocketFactory = Callable[[str], tuple[object, object]]
|
||||
_SEND_RETRY_INTERVAL_S = 0.01
|
||||
_CLOSE_TIMEOUT_S = 1.0
|
||||
@@ -92,7 +92,7 @@ class DirectLoadSnapshotSink:
|
||||
self._set_load_snapshot = set_load_snapshot
|
||||
|
||||
def observe(self, values: _LoadValues) -> None:
|
||||
num_running, num_waiting, num_active_pages, _, max_total_pages = values
|
||||
num_running, num_waiting, num_active_pages, _, max_total_pages, _ = values
|
||||
self._set_load_snapshot(
|
||||
num_running,
|
||||
num_waiting,
|
||||
@@ -232,12 +232,25 @@ class LoadSnapshotPublisher:
|
||||
|
||||
def _new_snapshot(self, values: _LoadValues) -> LoadSnapshot:
|
||||
self._sequence += 1
|
||||
(
|
||||
num_running,
|
||||
num_waiting,
|
||||
num_active_pages,
|
||||
num_used_pages,
|
||||
max_total_pages,
|
||||
num_resident_tokens,
|
||||
) = values
|
||||
return LoadSnapshot(
|
||||
self._epoch,
|
||||
self._sequence,
|
||||
self._dp_rank,
|
||||
*values,
|
||||
num_running,
|
||||
num_waiting,
|
||||
num_active_pages,
|
||||
num_used_pages,
|
||||
max_total_pages,
|
||||
self._valid_for_ms,
|
||||
num_resident_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -274,7 +287,7 @@ class LoadReporter:
|
||||
self._sample_stats = sample_stats
|
||||
self._enabled = enabled
|
||||
|
||||
def observe(self, stats: dict, num_running: int) -> None:
|
||||
def observe(self, stats: dict, num_running: int, num_resident_tokens: int) -> None:
|
||||
"""Publish a sample the caller already took."""
|
||||
self._publisher.observe(
|
||||
(
|
||||
@@ -283,14 +296,15 @@ class LoadReporter:
|
||||
stats["num_active_pages"],
|
||||
stats["num_active_pages"] + stats["num_cached_pages"],
|
||||
self._num_total_pages,
|
||||
num_resident_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
def sample_and_observe(self, num_running: int) -> None:
|
||||
def sample_and_observe(self, num_running: int, num_resident_tokens: int) -> None:
|
||||
"""Take a fresh scheduler sample and publish it."""
|
||||
if not self._enabled:
|
||||
return
|
||||
self.observe(self._sample_stats(), num_running)
|
||||
self.observe(self._sample_stats(), num_running, num_resident_tokens)
|
||||
|
||||
def close(self) -> None:
|
||||
self._publisher.close()
|
||||
@@ -375,6 +389,7 @@ class LoadSnapshotStore:
|
||||
snapshot.num_used_pages,
|
||||
snapshot.max_total_pages,
|
||||
snapshot.valid_for_ms,
|
||||
snapshot.num_resident_tokens,
|
||||
)
|
||||
self._snapshots[rank] = _StoredSnapshot(copied, now)
|
||||
return True
|
||||
@@ -423,6 +438,7 @@ class LoadSnapshotStore:
|
||||
snapshot.num_waiting_reqs,
|
||||
snapshot.num_active_pages,
|
||||
snapshot.num_used_pages,
|
||||
snapshot.num_resident_tokens,
|
||||
)
|
||||
):
|
||||
return False
|
||||
|
||||
@@ -93,6 +93,7 @@ def _snapshot(**overrides) -> LoadSnapshot:
|
||||
num_used_pages=5,
|
||||
max_total_pages=10,
|
||||
valid_for_ms=1_000,
|
||||
num_resident_tokens=0,
|
||||
)
|
||||
fields.update(overrides)
|
||||
return LoadSnapshot(**fields)
|
||||
|
||||
@@ -164,8 +164,12 @@ class _EventLoopHarness:
|
||||
note_round=lambda *, forwarded: self.trace.append("gc_round")
|
||||
)
|
||||
self.load_reporter = SimpleNamespace(
|
||||
observe=lambda _stats, _running: self.trace.append("observe_load"),
|
||||
sample_and_observe=lambda _running: self.trace.append("sample_load"),
|
||||
observe=lambda _stats, _running, _resident: self.trace.append(
|
||||
"observe_load"
|
||||
),
|
||||
sample_and_observe=lambda _running, _resident: self.trace.append(
|
||||
"sample_load"
|
||||
),
|
||||
close=lambda: self.trace.append("close_load"),
|
||||
)
|
||||
|
||||
@@ -192,6 +196,9 @@ class _EventLoopHarness:
|
||||
def _num_running(self) -> int:
|
||||
return 0
|
||||
|
||||
def _num_resident_tokens(self) -> int:
|
||||
return 0
|
||||
|
||||
def _record_scheduler_iteration_metrics(
|
||||
self, _stats, _num_iter_tokens: int
|
||||
) -> None:
|
||||
|
||||
@@ -63,6 +63,7 @@ def snapshot(dp_rank: int, **overrides) -> LoadSnapshot:
|
||||
num_used_pages=0,
|
||||
max_total_pages=100,
|
||||
valid_for_ms=1_000,
|
||||
num_resident_tokens=0,
|
||||
)
|
||||
fields.update(overrides)
|
||||
return LoadSnapshot(**fields)
|
||||
|
||||
@@ -119,7 +119,7 @@ def _roundtrip(obj):
|
||||
|
||||
def test_load_snapshot_round_trips_as_one_tagged_frame():
|
||||
"""The scheduler snapshot stays a standalone, single-frame IPC message."""
|
||||
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 6, 1_000)
|
||||
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 6, 1_000, 70_000)
|
||||
|
||||
frames = MsgpackEncoder().encode(snapshot)
|
||||
|
||||
|
||||
@@ -142,12 +142,13 @@ def observed_values(**overrides) -> dict:
|
||||
num_active_pages=3,
|
||||
num_used_pages=4,
|
||||
max_total_pages=10,
|
||||
num_resident_tokens=6,
|
||||
)
|
||||
values.update(overrides)
|
||||
return values
|
||||
|
||||
|
||||
def observed_tuple(**overrides) -> tuple[int, int, int, int, int]:
|
||||
def observed_tuple(**overrides) -> tuple[int, int, int, int, int, int]:
|
||||
values = observed_values(**overrides)
|
||||
return (
|
||||
values["num_running_reqs"],
|
||||
@@ -155,6 +156,7 @@ def observed_tuple(**overrides) -> tuple[int, int, int, int, int]:
|
||||
values["num_active_pages"],
|
||||
values["num_used_pages"],
|
||||
values["max_total_pages"],
|
||||
values["num_resident_tokens"],
|
||||
)
|
||||
|
||||
|
||||
@@ -191,6 +193,7 @@ def make_snapshot(**overrides) -> LoadSnapshot:
|
||||
num_used_pages=5,
|
||||
max_total_pages=10,
|
||||
valid_for_ms=int(VALID_FOR_SECONDS * 1_000),
|
||||
num_resident_tokens=6,
|
||||
)
|
||||
fields.update(overrides)
|
||||
return LoadSnapshot(**fields)
|
||||
@@ -228,6 +231,20 @@ def test_publisher_observe_accepts_one_scalar_tuple():
|
||||
publisher.close()
|
||||
|
||||
|
||||
def test_published_snapshot_carries_the_observed_resident_tokens():
|
||||
socket = RecordingSocket()
|
||||
publisher, _ = start_publisher(socket)
|
||||
try:
|
||||
publisher.observe(observed_tuple(num_resident_tokens=123_456))
|
||||
wait_until(lambda: socket.decoded_snapshots)
|
||||
|
||||
snapshot = socket.decoded_snapshots[0]
|
||||
assert (snapshot.dp_rank, snapshot.max_total_pages) == (2, 10)
|
||||
assert snapshot.num_resident_tokens == 123_456
|
||||
finally:
|
||||
publisher.close()
|
||||
|
||||
|
||||
def test_publisher_observe_does_not_encode_or_touch_zmq(monkeypatch):
|
||||
"""Scheduler observation performs only guarded scalar mailbox work."""
|
||||
factory_gate = threading.Event()
|
||||
@@ -396,6 +413,7 @@ def test_running_round_publishes_the_sample_the_loop_already_took(active, cached
|
||||
reporter.observe(
|
||||
{"num_queue_reqs": 3, "num_active_pages": active, "num_cached_pages": cached},
|
||||
num_running=2,
|
||||
num_resident_tokens=9_000,
|
||||
)
|
||||
|
||||
assert trace == [
|
||||
@@ -407,6 +425,7 @@ def test_running_round_publishes_the_sample_the_loop_already_took(active, cached
|
||||
num_active_pages=active,
|
||||
num_used_pages=active + cached,
|
||||
max_total_pages=20,
|
||||
num_resident_tokens=9_000,
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -421,15 +440,15 @@ def test_frozen_rounds_sample_for_themselves():
|
||||
scheduler_state = {"waiting": 0, "active": 0, "cached": 0}
|
||||
reporter = _reporter(trace, scheduler_state)
|
||||
|
||||
reporter.sample_and_observe(num_running=0)
|
||||
reporter.sample_and_observe(num_running=0, num_resident_tokens=0)
|
||||
scheduler_state.update(waiting=1, active=2, cached=2)
|
||||
reporter.sample_and_observe(num_running=0)
|
||||
reporter.sample_and_observe(num_running=1, num_resident_tokens=300)
|
||||
|
||||
assert trace == [
|
||||
"sample",
|
||||
("observe", (0, 0, 0, 0, 20)),
|
||||
("observe", (0, 0, 0, 0, 20, 0)),
|
||||
"sample",
|
||||
("observe", (0, 1, 2, 4, 20)),
|
||||
("observe", (1, 1, 2, 4, 20, 300)),
|
||||
]
|
||||
|
||||
|
||||
@@ -437,7 +456,7 @@ def test_non_reporting_rank_never_samples_the_scheduler():
|
||||
trace = []
|
||||
reporter = _reporter(trace, {"waiting": 0, "active": 0, "cached": 0}, enabled=False)
|
||||
|
||||
reporter.sample_and_observe(num_running=0)
|
||||
reporter.sample_and_observe(num_running=0, num_resident_tokens=0)
|
||||
|
||||
assert trace == []
|
||||
|
||||
@@ -490,6 +509,43 @@ def test_reporter_factory_selects_the_sink_for_each_mode(
|
||||
)
|
||||
|
||||
|
||||
def test_resident_tokens_sum_prompt_and_committed_output_of_every_request():
|
||||
pytest.importorskip("tokenspeed_scheduler")
|
||||
from tokenspeed.runtime.engine import event_loop as event_loop_module
|
||||
from tokenspeed.runtime.engine.generation_output_processor import (
|
||||
OutputProcesser,
|
||||
RequestState,
|
||||
)
|
||||
from tokenspeed.runtime.metrics.collector import EngineMetrics
|
||||
from tokenspeed.runtime.sampling.sampling_params import SamplingParams
|
||||
|
||||
output = OutputProcesser(
|
||||
send_to_tokenizer=lambda message: None,
|
||||
metrics=EngineMetrics(labels={}, enabled=False, reporters=[]),
|
||||
)
|
||||
decoding = RequestState(
|
||||
[1, 2, 3],
|
||||
SamplingParams(max_new_tokens=8),
|
||||
False,
|
||||
None,
|
||||
computes_prompt_logprobs=False,
|
||||
)
|
||||
decoding.output_ids.extend([7, 8])
|
||||
arriving = RequestState(
|
||||
[4] * 5,
|
||||
SamplingParams(max_new_tokens=8),
|
||||
False,
|
||||
None,
|
||||
computes_prompt_logprobs=False,
|
||||
)
|
||||
output.register("decoding", decoding)
|
||||
output.register("arriving", arriving)
|
||||
loop = event_loop_module.EventLoop.__new__(event_loop_module.EventLoop)
|
||||
loop.output_processor = output
|
||||
|
||||
assert loop._num_resident_tokens() == (3 + 2) + 5
|
||||
|
||||
|
||||
def test_event_loop_binds_the_output_setter_only_where_it_exists():
|
||||
"""Non-reporting ranks send through a NullSender with no snapshot setter,
|
||||
so the loop must not reach for it while wiring the reporter."""
|
||||
@@ -521,7 +577,7 @@ def test_event_loop_binds_the_output_setter_only_where_it_exists():
|
||||
|
||||
def test_load_snapshot_is_immutable_and_positional():
|
||||
"""Snapshots cannot be mutated after their wire values are assigned."""
|
||||
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 10, 1_000)
|
||||
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 10, 1_000, 6)
|
||||
|
||||
with pytest.raises(AttributeError):
|
||||
snapshot.sequence = 2
|
||||
@@ -589,6 +645,7 @@ def test_store_rejects_non_increasing_sequence_in_current_epoch():
|
||||
{"num_active_pages": 11},
|
||||
{"num_used_pages": 11},
|
||||
{"valid_for_ms": -1},
|
||||
{"num_resident_tokens": -1},
|
||||
],
|
||||
)
|
||||
def test_store_rejects_invalid_snapshot_values(overrides):
|
||||
|
||||
Reference in New Issue
Block a user