mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(server): report per-request prefill and decode timings in api responses * fix(server): take timings spans from the scheduler and gate them behind --enable-timings-report * fix(scheduler): count accepted drafts as min(n, emitted) and stamp reqstats with perf_counter
83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Dict, List
|
|
|
|
from .utils import deserialize_type, serialize_type
|
|
|
|
|
|
@dataclass
|
|
class BaseFrontendMsg:
|
|
@staticmethod
|
|
def encoder(msg: BaseFrontendMsg) -> Dict:
|
|
return serialize_type(msg)
|
|
|
|
@staticmethod
|
|
def decoder(json: Dict) -> BaseFrontendMsg:
|
|
return deserialize_type(globals(), json)
|
|
|
|
|
|
@dataclass
|
|
class BatchFrontendMsg(BaseFrontendMsg):
|
|
data: List[BaseFrontendMsg]
|
|
|
|
|
|
@dataclass
|
|
class UserReply(BaseFrontendMsg):
|
|
uid: int
|
|
incremental_output: str
|
|
finished: bool
|
|
prompt_tokens_delta: int = 0
|
|
completion_tokens_delta: int = 0
|
|
# Prefix-cache hit length (tokens of the prompt served from cache instead of
|
|
# recomputed). Arrives once, on the same reply as prompt_tokens_delta.
|
|
cached_tokens: int = 0
|
|
# KV page-pool usage snapshot (not-evictable used/total) for the shell status bar.
|
|
# 0/0 when not reported (e.g. the prompt-tokens reply, or owned-KV models with no
|
|
# shared page pool).
|
|
kv_used_pages: int = 0
|
|
kv_total_pages: int = 0
|
|
# GDN (mamba) state-pool slot usage (used/total) for hybrid models, else 0/0.
|
|
mamba_used_slots: int = 0
|
|
mamba_total_slots: int = 0
|
|
# Window (swa) pool token usage (used/total) for SWA models, else 0/0.
|
|
swa_used_tokens: int = 0
|
|
swa_total_tokens: int = 0
|
|
# Bytes the engine process holds on the GPU (torch reserved pool). 0 when not reported.
|
|
gpu_mem_bytes: int = 0
|
|
# Scheduler-measured per-request timings (queue/prompt/predicted spans, cache/prompt/
|
|
# predicted counts, draft counters), passthrough of DetokenizeMsg.timings: present on
|
|
# the terminal reply of a served request, None otherwise.
|
|
timings: dict | None = None
|
|
# Set (with finished=True) when a request failed before producing output — e.g. a chat
|
|
# template that the tokenizer cannot render, or a prompt that exceeds the KV budget the
|
|
# scheduler can serve. Carries a human-readable reason. Without this, such a request would
|
|
# send no `finished` reply at all and the API layer's wait_for_ack would hang forever.
|
|
error: str | None = None
|
|
# Machine-readable class for `error` (see ErrorReplyMsg.code), surfaced as OpenAI's error
|
|
# `code`. None when the failure has no specific class.
|
|
error_code: str | None = None
|
|
finish_reason: str | None = None
|
|
# The stop string that ended generation (Anthropic reports it as stop_reason='stop_sequence').
|
|
matched_stop: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class CacheRebuildReply(BaseFrontendMsg):
|
|
# detokenizer worker -> api server: result of a /v1/cache/rebuild request.
|
|
request_id: str
|
|
status: str # "ok" | "busy" | "failed"
|
|
moe_cache_size: int = 0
|
|
num_pages: int = 0
|
|
mamba_slots: int = 0
|
|
num_swa_pages: int = 0
|
|
error: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class FlushCacheReply(BaseFrontendMsg):
|
|
# detokenizer worker -> api server: result of a /v1/cache/flush request.
|
|
request_id: str
|
|
status: str # "ok" | "busy" | "timeout" | "unsupported" | "failed"
|
|
error: str | None = None
|