Files
Mukkesh Ganesh 03e5ea0672 feat(server): report per-request prefill and decode timings in api responses (#814)
* feat(server): report per-request prefill and decode timings in api responses

* fix(server): take timings spans from the scheduler and gate them behind --enable-timings-report

* fix(scheduler): count accepted drafts as min(n, emitted) and stamp reqstats with perf_counter
2026-11-03 14:44:25 -08:00

42 lines
1012 B
Python

from __future__ import annotations
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, List
import torch
from freetoken.core import ReqStats
if TYPE_CHECKING:
from freetoken.core import SamplingParams
from .prefill import ChunkedReq
@dataclass
class PendingReq:
uid: int
input_ids: torch.Tensor
sampling_params: SamplingParams
chunked_req: ChunkedReq | None = None
mm_items: list | None = None
mrope_positions_full: torch.Tensor | None = None
mrope_delta: int = 0
# timing/draft counters; add_one_req stamps enqueued_at on it, and every chunk
# Req built from this pending request shares the same instance
stats: ReqStats = field(default_factory=ReqStats)
@property
def input_len(self) -> int:
return len(self.input_ids)
@property
def output_len(self) -> int:
return self.sampling_params.max_tokens
@dataclass
class ScheduleResult:
reqs: List[PendingReq]
output_indices: List[torch.Tensor]