feat(engine): report resident tokens in the load snapshot

The load snapshot gains num_resident_tokens: the prompt plus committed
output tokens of every request the rank holds, summed over the output
processor's request states each time the loop samples load. Page counts
do not measure a rank's attention work (hybrid packing and state
reservations make them drift from the tokens attended), so a gateway
placing decode requests by context needs the token count itself.

The smg Python servicer forwards it as SchedulerLoad.num_resident_tokens.
The direct ZMQ lane does not carry it.

Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Enwei Zhu
2026-11-11 04:10:30 +00:00
co-authored by Claude Opus 5.5
parent bc58c927ad
commit 94b30e81ac
9 changed files with 116 additions and 18 deletions
+4
View File
@@ -425,6 +425,10 @@ Cached-only parents are `num_usable_pages - empty - active`. Active and cached
counts are disjoint, even when cache groups pack multiple blocks into a parent.
`LoadSnapshot.num_used_pages` sums active and cached parents to report all
resident occupancy, including evictable cache, rather than cache alone.
Parents do not measure a rank's attention work: hybrid packing and state
reservations make their count drift from the tokens attended. The snapshot
therefore also carries `num_resident_tokens`, the prompt plus committed
output of every request the rank holds, read from the output processor.
`available_lcm_blocks()` scans the pool and prefix indexes for reclaimability
and belongs in diagnostics and leak checks, never in the per-round sampler.
+12 -2
View File
@@ -1131,6 +1131,12 @@ class EventLoop:
def _num_running(self) -> int:
return len(self.output_processor.rid_to_state)
def _num_resident_tokens(self) -> int:
return sum(
state.input_length + state.output_length
for state in self.output_processor.rid_to_state.values()
)
def _request_context_length(self, rid: str) -> int:
"""Tokens ``rid`` holds right now: its prompt plus everything committed."""
state = self.output_processor.rid_to_state[rid]
@@ -1301,7 +1307,9 @@ class EventLoop:
forward_op = self._get_forward_op(execution_plan)
stats = self._get_scheduler_stats()
self.load_reporter.observe(stats, self._num_running())
self.load_reporter.observe(
stats, self._num_running(), self._num_resident_tokens()
)
num_iter_tokens = (
sum(forward_op.input_lengths) if forward_op is not None else 0
)
@@ -1431,7 +1439,9 @@ class EventLoop:
if self._pause.forward_blocked:
# Frozen rounds take no planning sample of their own; the
# idle sleep bounds this to one sample per millisecond.
self.load_reporter.sample_and_observe(self._num_running())
self.load_reporter.sample_and_observe(
self._num_running(), self._num_resident_tokens()
)
# Resolve a deferred abort/wait pause reply once in-flight work drains.
self._pause.maybe_finish_drain(self.scheduler)
@@ -1191,6 +1191,8 @@ class LoadSnapshot(
num_used_pages: int
max_total_pages: int
valid_for_ms: int
# Prompt plus committed output tokens summed over the rank's requests.
num_resident_tokens: int
class BlockReqType(Enum):
@@ -39,7 +39,7 @@ from tokenspeed.runtime.engine.io_struct import (
logger = logging.getLogger(__name__)
_LoadValues = tuple[int, int, int, int, int]
_LoadValues = tuple[int, int, int, int, int, int]
_SocketFactory = Callable[[str], tuple[object, object]]
_SEND_RETRY_INTERVAL_S = 0.01
_CLOSE_TIMEOUT_S = 1.0
@@ -92,7 +92,7 @@ class DirectLoadSnapshotSink:
self._set_load_snapshot = set_load_snapshot
def observe(self, values: _LoadValues) -> None:
num_running, num_waiting, num_active_pages, _, max_total_pages = values
num_running, num_waiting, num_active_pages, _, max_total_pages, _ = values
self._set_load_snapshot(
num_running,
num_waiting,
@@ -232,12 +232,25 @@ class LoadSnapshotPublisher:
def _new_snapshot(self, values: _LoadValues) -> LoadSnapshot:
self._sequence += 1
(
num_running,
num_waiting,
num_active_pages,
num_used_pages,
max_total_pages,
num_resident_tokens,
) = values
return LoadSnapshot(
self._epoch,
self._sequence,
self._dp_rank,
*values,
num_running,
num_waiting,
num_active_pages,
num_used_pages,
max_total_pages,
self._valid_for_ms,
num_resident_tokens,
)
@@ -274,7 +287,7 @@ class LoadReporter:
self._sample_stats = sample_stats
self._enabled = enabled
def observe(self, stats: dict, num_running: int) -> None:
def observe(self, stats: dict, num_running: int, num_resident_tokens: int) -> None:
"""Publish a sample the caller already took."""
self._publisher.observe(
(
@@ -283,14 +296,15 @@ class LoadReporter:
stats["num_active_pages"],
stats["num_active_pages"] + stats["num_cached_pages"],
self._num_total_pages,
num_resident_tokens,
)
)
def sample_and_observe(self, num_running: int) -> None:
def sample_and_observe(self, num_running: int, num_resident_tokens: int) -> None:
"""Take a fresh scheduler sample and publish it."""
if not self._enabled:
return
self.observe(self._sample_stats(), num_running)
self.observe(self._sample_stats(), num_running, num_resident_tokens)
def close(self) -> None:
self._publisher.close()
@@ -375,6 +389,7 @@ class LoadSnapshotStore:
snapshot.num_used_pages,
snapshot.max_total_pages,
snapshot.valid_for_ms,
snapshot.num_resident_tokens,
)
self._snapshots[rank] = _StoredSnapshot(copied, now)
return True
@@ -423,6 +438,7 @@ class LoadSnapshotStore:
snapshot.num_waiting_reqs,
snapshot.num_active_pages,
snapshot.num_used_pages,
snapshot.num_resident_tokens,
)
):
return False
@@ -93,6 +93,7 @@ def _snapshot(**overrides) -> LoadSnapshot:
num_used_pages=5,
max_total_pages=10,
valid_for_ms=1_000,
num_resident_tokens=0,
)
fields.update(overrides)
return LoadSnapshot(**fields)
+9 -2
View File
@@ -164,8 +164,12 @@ class _EventLoopHarness:
note_round=lambda *, forwarded: self.trace.append("gc_round")
)
self.load_reporter = SimpleNamespace(
observe=lambda _stats, _running: self.trace.append("observe_load"),
sample_and_observe=lambda _running: self.trace.append("sample_load"),
observe=lambda _stats, _running, _resident: self.trace.append(
"observe_load"
),
sample_and_observe=lambda _running, _resident: self.trace.append(
"sample_load"
),
close=lambda: self.trace.append("close_load"),
)
@@ -192,6 +196,9 @@ class _EventLoopHarness:
def _num_running(self) -> int:
return 0
def _num_resident_tokens(self) -> int:
return 0
def _record_scheduler_iteration_metrics(
self, _stats, _num_iter_tokens: int
) -> None:
@@ -63,6 +63,7 @@ def snapshot(dp_rank: int, **overrides) -> LoadSnapshot:
num_used_pages=0,
max_total_pages=100,
valid_for_ms=1_000,
num_resident_tokens=0,
)
fields.update(overrides)
return LoadSnapshot(**fields)
+1 -1
View File
@@ -119,7 +119,7 @@ def _roundtrip(obj):
def test_load_snapshot_round_trips_as_one_tagged_frame():
"""The scheduler snapshot stays a standalone, single-frame IPC message."""
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 6, 1_000)
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 6, 1_000, 70_000)
frames = MsgpackEncoder().encode(snapshot)
+64 -7
View File
@@ -142,12 +142,13 @@ def observed_values(**overrides) -> dict:
num_active_pages=3,
num_used_pages=4,
max_total_pages=10,
num_resident_tokens=6,
)
values.update(overrides)
return values
def observed_tuple(**overrides) -> tuple[int, int, int, int, int]:
def observed_tuple(**overrides) -> tuple[int, int, int, int, int, int]:
values = observed_values(**overrides)
return (
values["num_running_reqs"],
@@ -155,6 +156,7 @@ def observed_tuple(**overrides) -> tuple[int, int, int, int, int]:
values["num_active_pages"],
values["num_used_pages"],
values["max_total_pages"],
values["num_resident_tokens"],
)
@@ -191,6 +193,7 @@ def make_snapshot(**overrides) -> LoadSnapshot:
num_used_pages=5,
max_total_pages=10,
valid_for_ms=int(VALID_FOR_SECONDS * 1_000),
num_resident_tokens=6,
)
fields.update(overrides)
return LoadSnapshot(**fields)
@@ -228,6 +231,20 @@ def test_publisher_observe_accepts_one_scalar_tuple():
publisher.close()
def test_published_snapshot_carries_the_observed_resident_tokens():
socket = RecordingSocket()
publisher, _ = start_publisher(socket)
try:
publisher.observe(observed_tuple(num_resident_tokens=123_456))
wait_until(lambda: socket.decoded_snapshots)
snapshot = socket.decoded_snapshots[0]
assert (snapshot.dp_rank, snapshot.max_total_pages) == (2, 10)
assert snapshot.num_resident_tokens == 123_456
finally:
publisher.close()
def test_publisher_observe_does_not_encode_or_touch_zmq(monkeypatch):
"""Scheduler observation performs only guarded scalar mailbox work."""
factory_gate = threading.Event()
@@ -396,6 +413,7 @@ def test_running_round_publishes_the_sample_the_loop_already_took(active, cached
reporter.observe(
{"num_queue_reqs": 3, "num_active_pages": active, "num_cached_pages": cached},
num_running=2,
num_resident_tokens=9_000,
)
assert trace == [
@@ -407,6 +425,7 @@ def test_running_round_publishes_the_sample_the_loop_already_took(active, cached
num_active_pages=active,
num_used_pages=active + cached,
max_total_pages=20,
num_resident_tokens=9_000,
),
)
]
@@ -421,15 +440,15 @@ def test_frozen_rounds_sample_for_themselves():
scheduler_state = {"waiting": 0, "active": 0, "cached": 0}
reporter = _reporter(trace, scheduler_state)
reporter.sample_and_observe(num_running=0)
reporter.sample_and_observe(num_running=0, num_resident_tokens=0)
scheduler_state.update(waiting=1, active=2, cached=2)
reporter.sample_and_observe(num_running=0)
reporter.sample_and_observe(num_running=1, num_resident_tokens=300)
assert trace == [
"sample",
("observe", (0, 0, 0, 0, 20)),
("observe", (0, 0, 0, 0, 20, 0)),
"sample",
("observe", (0, 1, 2, 4, 20)),
("observe", (1, 1, 2, 4, 20, 300)),
]
@@ -437,7 +456,7 @@ def test_non_reporting_rank_never_samples_the_scheduler():
trace = []
reporter = _reporter(trace, {"waiting": 0, "active": 0, "cached": 0}, enabled=False)
reporter.sample_and_observe(num_running=0)
reporter.sample_and_observe(num_running=0, num_resident_tokens=0)
assert trace == []
@@ -490,6 +509,43 @@ def test_reporter_factory_selects_the_sink_for_each_mode(
)
def test_resident_tokens_sum_prompt_and_committed_output_of_every_request():
pytest.importorskip("tokenspeed_scheduler")
from tokenspeed.runtime.engine import event_loop as event_loop_module
from tokenspeed.runtime.engine.generation_output_processor import (
OutputProcesser,
RequestState,
)
from tokenspeed.runtime.metrics.collector import EngineMetrics
from tokenspeed.runtime.sampling.sampling_params import SamplingParams
output = OutputProcesser(
send_to_tokenizer=lambda message: None,
metrics=EngineMetrics(labels={}, enabled=False, reporters=[]),
)
decoding = RequestState(
[1, 2, 3],
SamplingParams(max_new_tokens=8),
False,
None,
computes_prompt_logprobs=False,
)
decoding.output_ids.extend([7, 8])
arriving = RequestState(
[4] * 5,
SamplingParams(max_new_tokens=8),
False,
None,
computes_prompt_logprobs=False,
)
output.register("decoding", decoding)
output.register("arriving", arriving)
loop = event_loop_module.EventLoop.__new__(event_loop_module.EventLoop)
loop.output_processor = output
assert loop._num_resident_tokens() == (3 + 2) + 5
def test_event_loop_binds_the_output_setter_only_where_it_exists():
"""Non-reporting ranks send through a NullSender with no snapshot setter,
so the loop must not reach for it while wiring the reporter."""
@@ -521,7 +577,7 @@ def test_event_loop_binds_the_output_setter_only_where_it_exists():
def test_load_snapshot_is_immutable_and_positional():
"""Snapshots cannot be mutated after their wire values are assigned."""
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 10, 1_000)
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 10, 1_000, 6)
with pytest.raises(AttributeError):
snapshot.sequence = 2
@@ -589,6 +645,7 @@ def test_store_rejects_non_increasing_sequence_in_current_epoch():
{"num_active_pages": 11},
{"num_used_pages": 11},
{"valid_for_ms": -1},
{"num_resident_tokens": -1},
],
)
def test_store_rejects_invalid_snapshot_values(overrides):