mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor: add engine and scheduler seams for multi-row decode steps * refactor: rename the graph steps argument to rows_per_req * refactor: drop VerifyInfo and require rows_per_req on verify batches * refactor: fix stale seam comments and read decode_rows_per_req directly * refactor: reword two GraphRunner comments for multi-row graphs * fix(engine): scale the GPU-decode MoE pool limits by decode_rows_per_req * refactor: drop the unread Context.moe_offload_cache field * test: trim the multi-row seam tests
43 lines
1.4 KiB
Python
43 lines
1.4 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Iterable, Set
|
|
|
|
from freetoken.core import Batch, Req
|
|
|
|
|
|
@dataclass
|
|
class DecodeManager:
|
|
page_size: int
|
|
# slots a decode step may hold past each request's committed length (provisional rows)
|
|
token_headroom: int = 0
|
|
running_reqs: Set[Req] = field(default_factory=set)
|
|
|
|
def filter_reqs(self, reqs: Iterable[Req]) -> None:
|
|
self.running_reqs = {req for req in self.running_reqs.union(reqs) if req.can_decode}
|
|
|
|
def remove_req(self, req: Req) -> None:
|
|
self.running_reqs.discard(req)
|
|
|
|
def abort_req(self, uid: int) -> Req | None:
|
|
for req in self.running_reqs:
|
|
if req.uid == uid:
|
|
self.running_reqs.remove(req)
|
|
return req
|
|
return None
|
|
|
|
@property
|
|
def inflight_tokens(self) -> int:
|
|
# each request's partly filled page, plus the provisional rows a decode step may add
|
|
tokens_reserved = (self.page_size - 1 + self.token_headroom) * len(self.running_reqs)
|
|
return sum(req.remain_len for req in self.running_reqs) + tokens_reserved
|
|
|
|
def schedule_next_batch(self) -> Batch | None:
|
|
if not self.runnable:
|
|
return None
|
|
return Batch(reqs=sorted(self.running_reqs, key=lambda req: req.uid), phase="decode")
|
|
|
|
@property
|
|
def runnable(self) -> bool:
|
|
return len(self.running_reqs) > 0
|