feat(engine): report pending uncached prefill tokens in the load snapshot

The load snapshot carried request counts but no prefill token work, so a
gateway balancing prefill ranks had to price every waiting or running
request at a mean prompt length. With agentic traffic, where long prompts
are mostly served from cache, that estimate is off by an order of
magnitude per request, and prefill ranks swing between idle and long
queues.

Scheduler::PendingPrefillTokens sums each request waiting for admission
as its prompt less the prefix the Device or shared L2 lookup already
holds, and each prefilling request as the prompt tokens still to compute.
The event loop publishes it as LoadSnapshot.num_waiting_uncached_tokens.

Requires the smg gRPC servicer to forward the field: its TokenSpeed load
conversion must copy num_waiting_uncached_tokens into SchedulerLoad and
mark the pushed load record as carrying queued token work. Without that
servicer change the gateway never sees the field and routing is unchanged.

DWDP16 prefill + DEP20 decode, agentic trace replay at 640 concurrent
sessions, one scored hour, gateway least_load reading the new field:
output in window 373.2 vs 358.3 tok/s/GPU (+4.2%), mean TTFT 14.3 vs
17.4 s, prefill batches underfilled with an empty queue 25.4% vs 38.8%.

Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com>
This commit is contained in:
Enwei Zhu
2026-11-14 16:11:49 +00:00
parent 3cc3e1ab62
commit 970f00f9c5
14 changed files with 101 additions and 13 deletions
+5
View File
@@ -429,6 +429,11 @@ Parents do not measure a rank's attention work: hybrid packing and state
reservations make their count drift from the tokens attended. The snapshot
therefore also carries `num_resident_tokens`, the prompt plus committed
output of every request the rank holds, read from the output processor.
A request count does not measure prefill work either: one waiting prompt may
need a few hundred new tokens, another a few hundred thousand. The snapshot
carries `num_waiting_uncached_tokens`, the scheduler's
`pending_prefill_tokens()`: each waiting prompt less what its cache lookup
found, plus the unscheduled rest of prompts in prefill.
`available_lcm_blocks()` scans the pool and prefix indexes for reclaimability
and belongs in diagnostics and leak checks, never in the per-round sampler.
@@ -1151,6 +1151,7 @@ class EventLoop:
self._scheduler_cache_geometry.num_usable_pages - empty - active
),
"num_queue_reqs": self.scheduler.waiting_size(),
"num_pending_prefill_tokens": self.scheduler.pending_prefill_tokens(),
}
def _record_scheduler_iteration_metrics(
@@ -1193,6 +1193,9 @@ class LoadSnapshot(
valid_for_ms: int
# Prompt plus committed output tokens summed over the rank's requests.
num_resident_tokens: int
# Prompt tokens still to compute: waiting prompts less their cache hit, plus
# the unscheduled rest of prompts in prefill (the gateway's queued work).
num_waiting_uncached_tokens: int
class BlockReqType(Enum):
@@ -39,7 +39,7 @@ from tokenspeed.runtime.engine.io_struct import (
logger = logging.getLogger(__name__)
_LoadValues = tuple[int, int, int, int, int, int]
_LoadValues = tuple[int, int, int, int, int, int, int]
_SocketFactory = Callable[[str], tuple[object, object]]
_SEND_RETRY_INTERVAL_S = 0.01
_CLOSE_TIMEOUT_S = 1.0
@@ -92,7 +92,7 @@ class DirectLoadSnapshotSink:
self._set_load_snapshot = set_load_snapshot
def observe(self, values: _LoadValues) -> None:
num_running, num_waiting, num_active_pages, _, max_total_pages, _ = values
num_running, num_waiting, num_active_pages, _, max_total_pages, _, _ = values
self._set_load_snapshot(
num_running,
num_waiting,
@@ -239,6 +239,7 @@ class LoadSnapshotPublisher:
num_used_pages,
max_total_pages,
num_resident_tokens,
num_pending_prefill_tokens,
) = values
return LoadSnapshot(
self._epoch,
@@ -251,6 +252,7 @@ class LoadSnapshotPublisher:
max_total_pages,
self._valid_for_ms,
num_resident_tokens,
num_pending_prefill_tokens,
)
@@ -297,6 +299,7 @@ class LoadReporter:
stats["num_active_pages"] + stats["num_cached_pages"],
self._num_total_pages,
num_resident_tokens,
stats["num_pending_prefill_tokens"],
)
)
@@ -390,6 +393,7 @@ class LoadSnapshotStore:
snapshot.max_total_pages,
snapshot.valid_for_ms,
snapshot.num_resident_tokens,
snapshot.num_waiting_uncached_tokens,
)
self._snapshots[rank] = _StoredSnapshot(copied, now)
return True
@@ -94,6 +94,7 @@ def _snapshot(**overrides) -> LoadSnapshot:
max_total_pages=10,
valid_for_ms=1_000,
num_resident_tokens=0,
num_waiting_uncached_tokens=0,
)
fields.update(overrides)
return LoadSnapshot(**fields)
@@ -64,6 +64,7 @@ def snapshot(dp_rank: int, **overrides) -> LoadSnapshot:
max_total_pages=100,
valid_for_ms=1_000,
num_resident_tokens=0,
num_waiting_uncached_tokens=0,
)
fields.update(overrides)
return LoadSnapshot(**fields)
+1 -1
View File
@@ -119,7 +119,7 @@ def _roundtrip(obj):
def test_load_snapshot_round_trips_as_one_tagged_frame():
"""The scheduler snapshot stays a standalone, single-frame IPC message."""
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 6, 1_000, 70_000)
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 6, 1_000, 70_000, 12_000)
frames = MsgpackEncoder().encode(snapshot)
+35 -9
View File
@@ -143,12 +143,13 @@ def observed_values(**overrides) -> dict:
num_used_pages=4,
max_total_pages=10,
num_resident_tokens=6,
num_waiting_uncached_tokens=7,
)
values.update(overrides)
return values
def observed_tuple(**overrides) -> tuple[int, int, int, int, int, int]:
def observed_tuple(**overrides) -> tuple[int, int, int, int, int, int, int]:
values = observed_values(**overrides)
return (
values["num_running_reqs"],
@@ -157,6 +158,7 @@ def observed_tuple(**overrides) -> tuple[int, int, int, int, int, int]:
values["num_used_pages"],
values["max_total_pages"],
values["num_resident_tokens"],
values["num_waiting_uncached_tokens"],
)
@@ -194,6 +196,7 @@ def make_snapshot(**overrides) -> LoadSnapshot:
max_total_pages=10,
valid_for_ms=int(VALID_FOR_SECONDS * 1_000),
num_resident_tokens=6,
num_waiting_uncached_tokens=7,
)
fields.update(overrides)
return LoadSnapshot(**fields)
@@ -245,6 +248,18 @@ def test_published_snapshot_carries_the_observed_resident_tokens():
publisher.close()
def test_published_snapshot_carries_the_pending_prompt_work():
socket = RecordingSocket()
publisher, _ = start_publisher(socket)
try:
publisher.observe(observed_tuple(num_waiting_uncached_tokens=48_000))
wait_until(lambda: socket.decoded_snapshots)
assert socket.decoded_snapshots[0].num_waiting_uncached_tokens == 48_000
finally:
publisher.close()
def test_publisher_observe_does_not_encode_or_touch_zmq(monkeypatch):
"""Scheduler observation performs only guarded scalar mailbox work."""
factory_gate = threading.Event()
@@ -394,6 +409,7 @@ def _reporter(trace, scheduler_state, *, enabled=True):
"num_queue_reqs": scheduler_state["waiting"],
"num_active_pages": scheduler_state["active"],
"num_cached_pages": scheduler_state["cached"],
"num_pending_prefill_tokens": scheduler_state["pending"],
}
return load_snapshot_module.LoadReporter(
@@ -408,10 +424,17 @@ def _reporter(trace, scheduler_state, *, enabled=True):
def test_running_round_publishes_the_sample_the_loop_already_took(active, cached):
"""The active path never samples again: the loop passes its own stats."""
trace = []
reporter = _reporter(trace, {"waiting": 3, "active": active, "cached": cached})
reporter = _reporter(
trace, {"waiting": 3, "active": active, "cached": cached, "pending": 1_500}
)
reporter.observe(
{"num_queue_reqs": 3, "num_active_pages": active, "num_cached_pages": cached},
{
"num_queue_reqs": 3,
"num_active_pages": active,
"num_cached_pages": cached,
"num_pending_prefill_tokens": 1_500,
},
num_running=2,
num_resident_tokens=9_000,
)
@@ -426,6 +449,7 @@ def test_running_round_publishes_the_sample_the_loop_already_took(active, cached
num_used_pages=active + cached,
max_total_pages=20,
num_resident_tokens=9_000,
num_waiting_uncached_tokens=1_500,
),
)
]
@@ -437,24 +461,26 @@ def test_frozen_rounds_sample_for_themselves():
test_unchanged_values_only_emit_a_heartbeat), and the idle sleep bounds
the rate to one sample per millisecond."""
trace = []
scheduler_state = {"waiting": 0, "active": 0, "cached": 0}
scheduler_state = {"waiting": 0, "active": 0, "cached": 0, "pending": 0}
reporter = _reporter(trace, scheduler_state)
reporter.sample_and_observe(num_running=0, num_resident_tokens=0)
scheduler_state.update(waiting=1, active=2, cached=2)
scheduler_state.update(waiting=1, active=2, cached=2, pending=700)
reporter.sample_and_observe(num_running=1, num_resident_tokens=300)
assert trace == [
"sample",
("observe", (0, 0, 0, 0, 20, 0)),
("observe", (0, 0, 0, 0, 20, 0, 0)),
"sample",
("observe", (1, 1, 2, 4, 20, 300)),
("observe", (1, 1, 2, 4, 20, 300, 700)),
]
def test_non_reporting_rank_never_samples_the_scheduler():
trace = []
reporter = _reporter(trace, {"waiting": 0, "active": 0, "cached": 0}, enabled=False)
reporter = _reporter(
trace, {"waiting": 0, "active": 0, "cached": 0, "pending": 0}, enabled=False
)
reporter.sample_and_observe(num_running=0, num_resident_tokens=0)
@@ -577,7 +603,7 @@ def test_event_loop_binds_the_output_setter_only_where_it_exists():
def test_load_snapshot_is_immutable_and_positional():
"""Snapshots cannot be mutated after their wire values are assigned."""
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 10, 1_000, 6)
snapshot = LoadSnapshot("boot-a", 1, 0, 2, 3, 4, 5, 10, 1_000, 6, 7)
with pytest.raises(AttributeError):
snapshot.sequence = 2
@@ -677,6 +677,9 @@ NB_MODULE(tokenspeed_scheduler_ext, m) {
"Count requests waiting for their PD bootstrap handshake.")
.def("waiting_size", &tokenspeed::Scheduler::WaitingSize,
"Count Submitted, Prefetching, Retracted and Restoring requests awaiting admission or their restore.")
.def("pending_prefill_tokens", &tokenspeed::Scheduler::PendingPrefillTokens,
"Prompt tokens still to compute: waiting prompts less what the caches held for them at their lookup, "
"plus the unscheduled rest of prompts in prefill.")
.def("retracted_size", &tokenspeed::Scheduler::RetractedSize,
"Count requests suspended with an image (Retracted or Restoring); a weight update must not flush "
"while this is non-zero.")
@@ -99,6 +99,9 @@ struct AdmissionResources {
// pages reach: past the hit, a junction the first chunk is cut at.
bool pool_lookup_asked{false};
std::int32_t pool_kv_tokens{0};
// The prompt tokens the Device or pool cache held at the lookup: what the
// load report subtracts from the prompt as work already done.
std::int32_t cached_prefix_estimate{0};
std::optional<std::vector<std::string>> prefix_hashes;
};
@@ -44,7 +44,10 @@ std::vector<SharedL2Lookup> Scheduler::SharedL2Lookups() {
static_cast<std::size_t>(max_tokens / config_.prefix_granularity)));
const CacheCoordinator::PromptPrefix prompt{
.prefix_hashes = hashes, .tokens = request->TokenSlice(0, max_tokens), .max_tokens = max_tokens};
if (max_tokens == 0 || coordinator_.ProbePromptPrefix(prompt).device.num_common_tokens >= max_tokens) {
const std::int32_t device_hit =
max_tokens == 0 ? 0 : coordinator_.ProbePromptPrefix(prompt).device.num_common_tokens;
admission->cached_prefix_estimate = device_hit;
if (max_tokens == 0 || device_hit >= max_tokens) {
// Nothing the pool could add.
shared_l2_->ApplyLookup(SharedL2LookupResult{.request_id = request->Id()}, ContentHash{});
continue;
@@ -66,6 +69,7 @@ std::vector<SharedL2ProxyPage> Scheduler::ApplySharedL2Lookups(const std::vector
_assert(admission != nullptr && admission->pool_lookup_asked && !shared_l2_->LookedUp(result.request_id),
"a shared L2 lookup answers a waiting request that asked");
admission->pool_kv_tokens = result.kv_tokens;
admission->cached_prefix_estimate = std::max(admission->cached_prefix_estimate, result.num_tokens);
ContentHash prior;
if (result.tail) {
const auto full_pages = (result.num_tokens - result.tail->tail_tokens) / config_.prefix_granularity;
@@ -396,6 +396,18 @@ std::size_t Scheduler::WaitingSize() const {
}));
}
std::int64_t Scheduler::PendingPrefillTokens() const {
std::int64_t tokens = 0;
for (const std::unique_ptr<Request>& request : requests_) {
if (const fsm::AdmissionResources* admission = request->AdmissionResourcesPtr()) {
tokens += std::max(0, request->PrefillSize() - admission->cached_prefix_estimate);
} else if (request->Is<fsm::Prefilling>()) {
tokens += std::max(0, request->PrefillSize() - request->NumComputedTokens());
}
}
return tokens;
}
std::size_t Scheduler::RetractedSize() const {
return static_cast<std::size_t>(std::ranges::count_if(requests_, [](const std::unique_ptr<Request>& request) {
return request->IsAnyOf<fsm::Retracted, fsm::Restoring>();
@@ -90,6 +90,9 @@ public:
// Retracted (waiting to be restored) and Restoring (their copy back is in
// flight) -- which hold no schedulable work until they resume.
std::size_t WaitingSize() const;
// Prompt tokens still to compute: each waiting prompt less what the caches
// held for it at its lookup, plus the unscheduled rest of prompts in prefill.
std::int64_t PendingPrefillTokens() const;
std::size_t DecodingSize() const;
std::size_t PrefillSize() const;
std::size_t AwaitingDestinationSize() const;
@@ -214,6 +214,28 @@ def test_a_rank_resumes_from_the_pool_where_another_rank_published():
assert pool.stats()["pinned_blocks"] == 0
def test_pending_prefill_tokens_count_each_prompt_less_its_cache_hit():
"""The load report's queued work: a waiting prompt less what its lookup
found cached, and a prompt in prefill less what is already scheduled."""
pool = owner()
writer, reader = scheduler(), scheduler()
writer_rt, reader_rt = Runtime(pool, 1), Runtime(pool, 2)
prompt = list(range(9))
publish(writer, writer_rt, "a", prompt)
reader.submit_requests([_spec("b", prompt + [50, 51])])
assert reader.pending_prefill_tokens() == len(prompt) + 2
reader_rt.lookup(reader)
assert reader.pending_prefill_tokens() == 2
reader_rt.plan(reader, lookup=False)
assert reader.pending_prefill_tokens() == 0
cold, cold_rt = scheduler(), Runtime(pool, 3)
cold.submit_requests([_spec("c", list(range(200, 300)))])
cold_rt.plan(cold)
assert cold.pending_prefill_tokens() == 100 - 64
def test_a_waiting_prompt_is_admitted_only_after_its_lookup():
pool = owner()
writer, reader = scheduler(), scheduler()