mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(server): report per-request prefill and decode timings in api responses * fix(server): take timings spans from the scheduler and gate them behind --enable-timings-report * fix(scheduler): count accepted drafts as min(n, emitted) and stamp reqstats with perf_counter
138 lines
4.6 KiB
Python
138 lines
4.6 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any, Dict, List
|
|
|
|
from freetoken.core import SamplingParams
|
|
|
|
from .utils import deserialize_type, serialize_type
|
|
|
|
|
|
@dataclass
|
|
class BaseTokenizerMsg:
|
|
@staticmethod
|
|
def encoder(msg: BaseTokenizerMsg) -> Dict:
|
|
return serialize_type(msg)
|
|
|
|
@staticmethod
|
|
def decoder(json: Dict) -> BaseTokenizerMsg:
|
|
return deserialize_type(globals(), json)
|
|
|
|
|
|
@dataclass
|
|
class BatchTokenizerMsg(BaseTokenizerMsg):
|
|
data: List[BaseTokenizerMsg]
|
|
|
|
|
|
@dataclass
|
|
class DetokenizeMsg(BaseTokenizerMsg):
|
|
uid: int
|
|
# tokens committed this step, in order: one per decode step, up to k+1 per speculative verify
|
|
next_tokens: List[int]
|
|
finished: bool
|
|
finish_reason: str | None = None
|
|
# The stop string that ended generation (if any), so the detokenizer can trim it
|
|
# and everything after it from the final output.
|
|
matched_stop: str | None = None
|
|
# The request's stop strings (None when it has none), so the detokenizer can hold back
|
|
# a trailing partial-stop prefix instead of streaming it and then needing to retract.
|
|
stop_strs: list[str] | None = None
|
|
# KV page-pool usage snapshot at this step (not-evictable used/total), passed
|
|
# through to the frontend for the shell status bar. 0/0 for owned-KV models.
|
|
kv_used_pages: int = 0
|
|
kv_total_pages: int = 0
|
|
# GDN (mamba) state-pool slot usage (used/total) for hybrid models, else 0/0.
|
|
mamba_used_slots: int = 0
|
|
mamba_total_slots: int = 0
|
|
# Window (swa) pool token usage (used/total) for SWA models, else 0/0.
|
|
swa_used_tokens: int = 0
|
|
swa_total_tokens: int = 0
|
|
# Bytes this engine process holds on the GPU (torch reserved pool). 0 on CPU.
|
|
gpu_mem_bytes: int = 0
|
|
# Scheduler-measured per-request timings (ReqStats.finish) on the terminal reply
|
|
# only; None on intermediate replies, errors and aborts.
|
|
timings: dict | None = None
|
|
|
|
|
|
@dataclass
|
|
class PromptAdmittedMsg(BaseTokenizerMsg):
|
|
"""Scheduler -> tokenizer accounting signal for an admitted prompt.
|
|
|
|
Emitted exactly once, after the request's first prefill batch has been prepared
|
|
successfully. ``prompt_tokens`` is the complete prompt length, even when the
|
|
prefill itself is chunked. ``cached_tokens`` is the prefix-cache hit — the subset
|
|
of ``prompt_tokens`` served from cache instead of recomputed.
|
|
"""
|
|
|
|
uid: int
|
|
prompt_tokens: int
|
|
cached_tokens: int = 0
|
|
|
|
|
|
@dataclass
|
|
class TokenizeMsg(BaseTokenizerMsg):
|
|
uid: int
|
|
text: str | List[Dict[str, Any]]
|
|
sampling_params: SamplingParams
|
|
chat_template_kwargs: Dict[str, Any] | None = None
|
|
tools: List[Dict[str, Any]] | None = None
|
|
images: List[bytes] | None = None
|
|
inline_system_policy: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class AbortMsg(BaseTokenizerMsg):
|
|
uid: int
|
|
|
|
|
|
@dataclass
|
|
class CacheRebuildMsg(BaseTokenizerMsg):
|
|
# api server -> tokenizer worker (pure passthrough to CacheRebuildBackendMsg).
|
|
request_id: str
|
|
moe_cache_size: int | None = None
|
|
num_pages: int | None = None
|
|
num_mamba_slots: int | None = None
|
|
num_swa_pages: int | None = None
|
|
mode: str = "if_idle"
|
|
|
|
|
|
@dataclass
|
|
class CacheRebuildResultMsg(BaseTokenizerMsg):
|
|
# scheduler -> detokenizer worker (passthrough to CacheRebuildReply).
|
|
request_id: str
|
|
status: str # "ok" | "busy" | "failed"
|
|
moe_cache_size: int = 0
|
|
num_pages: int = 0
|
|
mamba_slots: int = 0
|
|
num_swa_pages: int = 0
|
|
error: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class FlushCacheMsg(BaseTokenizerMsg):
|
|
# api server -> tokenizer worker (pure passthrough to FlushCacheBackendMsg).
|
|
request_id: str
|
|
timeout: float = 0.0
|
|
|
|
|
|
@dataclass
|
|
class FlushCacheResultMsg(BaseTokenizerMsg):
|
|
# scheduler -> detokenizer worker (passthrough to FlushCacheReply).
|
|
request_id: str
|
|
status: str # "ok" | "busy" | "timeout" | "unsupported"
|
|
error: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class ErrorReplyMsg(BaseTokenizerMsg):
|
|
# scheduler -> tokenizer/detokenizer worker -> frontend: a request the scheduler cannot
|
|
# serve (e.g. its prompt exceeds the KV budget). The worker translates it into a terminal
|
|
# UserReply(error=...) so the client gets a prompt error instead of hanging on a `finished`
|
|
# reply that will never come. Mirrors the CacheRebuildResultMsg passthrough pattern.
|
|
uid: int
|
|
error: str
|
|
# Stable machine-readable reason, mirrored onto the wire as OpenAI's error `code` so a client
|
|
# can react without parsing prose. Today only "context_length_exceeded" (prompt longer than
|
|
# the servable context); None = no specific class, the message is all there is.
|
|
code: str | None = None
|