mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(server): report per-request prefill and decode timings in api responses * fix(server): take timings spans from the scheduler and gate them behind --enable-timings-report * fix(scheduler): count accepted drafts as min(n, emitted) and stamp reqstats with perf_counter
884 lines
37 KiB
Python
884 lines
37 KiB
Python
"""Protocol-neutral generation core.
|
|
|
|
The narrow interface every wire protocol (OpenAI chat, Anthropic messages, OpenAI
|
|
Responses) sits on. A protocol adapter converts its request into a ``GenSpec``
|
|
(messages + sampling + tools), submits it, and formats the resulting semantic
|
|
events (``GenEvent``) / ``GenResult`` into its own wire shape. This mirrors vLLM's
|
|
split of per-request ``to_sampling_params`` + shared preprocess + a neutral
|
|
``SamplingParams`` — no wire request type reaches the engine path.
|
|
|
|
This module is imported by ``openai_api`` / ``anthropic_api`` / ``responses_api``;
|
|
it depends on none of them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import time
|
|
from collections.abc import AsyncIterator
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from . import request_ring
|
|
from freetoken.core import SamplingParams
|
|
from freetoken.message import TokenizeMsg
|
|
from freetoken.mm.media import collect_image_refs, fetch_image_bytes, image_reject_reason
|
|
from freetoken.tokenizer.tokenize import resolve_thinking_mode
|
|
from freetoken.tokenizer.inline_system import InlineSystemError
|
|
|
|
try:
|
|
# Chat templates render through jinja2 (a transformers dependency): a TemplateError means
|
|
# the template rejected the specific conversation (bad role ordering, an unmatched
|
|
# tool_result, an explicit raise_exception) — an input-driven, client-classifiable failure.
|
|
from jinja2 import TemplateError as _TemplateError
|
|
except Exception: # pragma: no cover — jinja2 always ships with transformers
|
|
_TemplateError = ()
|
|
|
|
from .function_call_parser import FunctionCallParser, TOOLS_TAG_LIST, ToolCallItem
|
|
from .reasoning_parser import (
|
|
DSV4_SPECIAL_TOKENS,
|
|
ReasoningParser,
|
|
build_reasoning_parser,
|
|
strip_special_tokens,
|
|
)
|
|
|
|
|
|
class GenerationError(ValueError):
|
|
"""A request failed before producing output (surfaced via ``UserReply.error`` — e.g. a
|
|
chat template the tokenizer cannot render, or a prompt that exceeds the KV budget). Each
|
|
adapter turns this into its own wire-level error instead of hanging on a reply that never
|
|
arrives. ``code`` carries the stable class (``UserReply.error_code``) when the failure has
|
|
one, so an adapter can emit it as OpenAI's error ``code`` and clients need not parse prose.
|
|
|
|
Subclasses ValueError so every adapter's invalid-request handler maps it to a 400."""
|
|
|
|
def __init__(self, message: str, code: str | None = None):
|
|
super().__init__(message)
|
|
self.code = code
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Protocol-neutral generation events.
|
|
#
|
|
# generate_events() / generate_full() yield these instead of OpenAI wire chunks.
|
|
# The OpenAI, Anthropic, and Responses streamers are thin formatters over them,
|
|
# so generation/parsing logic lives in exactly one place and no adapter has to
|
|
# re-parse a serialized OpenAI stream.
|
|
# --------------------------------------------------------------------------- #
|
|
@dataclass
|
|
class ReasoningDelta:
|
|
text: str
|
|
|
|
|
|
@dataclass
|
|
class ContentDelta:
|
|
text: str
|
|
|
|
|
|
@dataclass
|
|
class ToolCallStart:
|
|
"""A tool call opened mid-stream: its name is known, arguments follow as
|
|
ToolCallArgsDelta fragments, and the matching ToolCallsDelta closes it with the
|
|
authoritative final arguments. tool_index is the output ordinal (0, 1, ...).
|
|
args_prefix_stable tells adapters whether the fragments are safe to forward to
|
|
clients that concatenate them (else: send full args once at close)."""
|
|
|
|
tool_index: int
|
|
name: str | None
|
|
args_prefix_stable: bool = True
|
|
|
|
|
|
@dataclass
|
|
class ToolCallArgsDelta:
|
|
"""An incremental fragment of the open call's arguments JSON. Fragments always
|
|
concatenate to a prefix of the final arguments (detectors skip non-prefix
|
|
diffs), so adapters may stream them and top up the remainder at close."""
|
|
|
|
tool_index: int
|
|
fragment: str
|
|
|
|
|
|
@dataclass
|
|
class ToolCallsDelta:
|
|
"""Complete call(s). On the streaming path this closes a ToolCallStart-opened
|
|
call (same tool_index) carrying the final arguments; on the buffered fallback
|
|
path it arrives standalone, without a preceding start/args events."""
|
|
|
|
calls: list[ToolCallItem]
|
|
|
|
|
|
@dataclass
|
|
class GenDone:
|
|
finish_reason: str
|
|
prompt_tokens: int
|
|
completion_tokens: int
|
|
matched_stop: str | None = None
|
|
cached_tokens: int = 0
|
|
timings: dict[str, Any] | None = None
|
|
|
|
|
|
GenEvent = ReasoningDelta | ContentDelta | ToolCallStart | ToolCallArgsDelta | ToolCallsDelta | GenDone
|
|
|
|
|
|
@dataclass
|
|
class GenResult:
|
|
reasoning: str
|
|
content: str
|
|
tool_calls: list[ToolCallItem]
|
|
finish_reason: str
|
|
prompt_tokens: int
|
|
completion_tokens: int
|
|
matched_stop: str | None = None
|
|
cached_tokens: int = 0
|
|
timings: dict[str, Any] | None = None
|
|
|
|
|
|
def timings_block(state: Any, timings: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
"""llama.cpp's per-request timings object (server_slot_stats::to_json, b10408+) as
|
|
frontends like llama-swap read it, plus queue_ms as a FreeToken extension. The spans
|
|
and counts are measured in the scheduler (ReqStats) and arrive on the terminal ack:
|
|
queue_ms covers waiting-queue admission to the first prefill chunk, prompt_ms the
|
|
first chunk to the first sampled token on the host, predicted_ms the first to the
|
|
last committed token. prompt_n counts only the evaluated tokens and the decode rates
|
|
divide by predicted_n - 1 steps (the first token rides the last prompt batch),
|
|
matching llama.cpp's arithmetic. Values are not rounded. Off unless
|
|
--enable-timings-report: usage.prompt_tokens - prompt_n would otherwise still
|
|
reveal the prefix-cache hit that --enable-cache-report hides."""
|
|
if timings is None:
|
|
return None
|
|
config = getattr(state, "config", None) if state is not None else None
|
|
if not getattr(config, "enable_timings_report", False):
|
|
return None
|
|
prompt_n = timings["prompt_n"]
|
|
predicted_n = timings["predicted_n"]
|
|
prompt_ms = timings["prompt_ms"]
|
|
predicted_ms = timings["predicted_ms"]
|
|
gen_steps = max(predicted_n - 1, 0)
|
|
block: dict[str, Any] = {
|
|
"cache_n": timings["cache_n"],
|
|
"prompt_n": prompt_n,
|
|
"prompt_ms": prompt_ms,
|
|
"prompt_per_token_ms": prompt_ms / prompt_n if prompt_n > 0 else 0.0,
|
|
"prompt_per_second": 1e3 / prompt_ms * prompt_n if prompt_ms > 0 else 0.0,
|
|
"predicted_n": predicted_n,
|
|
"predicted_ms": predicted_ms,
|
|
"predicted_per_token_ms": predicted_ms / gen_steps if gen_steps > 0 else 0.0,
|
|
"predicted_per_second": 1e3 / predicted_ms * gen_steps if predicted_ms > 0 else 0.0,
|
|
"queue_ms": timings["queue_ms"],
|
|
}
|
|
if timings.get("draft_n", 0) > 0:
|
|
block["draft_n"] = timings["draft_n"]
|
|
block["draft_n_accepted"] = timings["draft_n_accepted"]
|
|
return block
|
|
|
|
|
|
@dataclass
|
|
class GenSpec:
|
|
"""Protocol-neutral 'what to generate'. Each wire protocol converts its request
|
|
into a GenSpec (like vLLM's to_sampling_params + _preprocess_chat) and the
|
|
primitive consumes ONLY this — no wire request type reaches the engine path."""
|
|
|
|
messages: list[dict[str, Any]] # normalized, template-ready
|
|
sampling_params: SamplingParams
|
|
chat_template_kwargs: dict[str, Any] = field(default_factory=dict)
|
|
template_tools: list[dict[str, Any]] | None = None # tools the model sees (TokenizeMsg.tools)
|
|
parser_tools: list[dict[str, Any]] | None = None # tools for FunctionCallParser; None disables parsing
|
|
inline_system_policy: str | None = None
|
|
|
|
@property
|
|
def parse_tools(self) -> bool:
|
|
return self.parser_tools is not None
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Wire-neutral builders (shared by every protocol's request->GenSpec converter).
|
|
# --------------------------------------------------------------------------- #
|
|
# Default max output (decode) tokens when a request omits one. Overridable per server via
|
|
# --max-output-tokens (the Responses adapter passes that through); clamped to the remaining
|
|
# context by the scheduler regardless.
|
|
DEFAULT_MAX_OUTPUT_TOKENS = 32768
|
|
|
|
|
|
def resolve_sampling(
|
|
*,
|
|
temperature: float | None,
|
|
top_k: int | None,
|
|
top_p: float | None,
|
|
max_tokens: int | None,
|
|
ignore_eos: bool,
|
|
model_sampling: dict[str, Any],
|
|
stop: str | list[str] | None = None,
|
|
default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,
|
|
presence_penalty: float = 0.0,
|
|
frequency_penalty: float = 0.0,
|
|
) -> SamplingParams:
|
|
"""Map a protocol's sampling fields onto the engine's neutral SamplingParams,
|
|
filling unspecified fields from the checkpoint's recommended defaults."""
|
|
|
|
def pick(value, key, framework):
|
|
return value if value is not None else model_sampling.get(key, framework)
|
|
|
|
stop_list = [stop] if isinstance(stop, str) else list(stop or [])
|
|
# `is not None`, not truthiness: an explicit max_tokens=0 must not read as "unset" and
|
|
# silently become the 32k default. The engine cannot serve a zero-token budget either
|
|
# (the request would never become decodable, so the client would wait forever), so a
|
|
# non-positive value is a client error.
|
|
if max_tokens is not None and max_tokens < 1:
|
|
raise ValueError(f"max_tokens must be at least 1, got {max_tokens}")
|
|
return SamplingParams(
|
|
ignore_eos=ignore_eos,
|
|
max_tokens=default_max_tokens if max_tokens is None else max_tokens,
|
|
temperature=pick(temperature, "temperature", 0.0),
|
|
top_k=pick(top_k, "top_k", -1),
|
|
top_p=pick(top_p, "top_p", 1.0),
|
|
stop_strs=[s for s in stop_list if s], # drop empty strings (would match everything)
|
|
presence_penalty=presence_penalty,
|
|
frequency_penalty=frequency_penalty,
|
|
)
|
|
|
|
|
|
def render_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Normalize OpenAI-shaped message dicts for the chat template: flatten text
|
|
content parts to a string and decode tool-call arguments from JSON. Raises
|
|
ValueError on a non-text content part (text-only server). Shared by all adapters."""
|
|
return [_render_message(m) for m in messages]
|
|
|
|
|
|
def _render_message(message: dict[str, Any]) -> dict[str, Any]:
|
|
m = dict(message)
|
|
content = m.get("content")
|
|
if isinstance(content, list):
|
|
m["content"] = _flatten_text_parts(content)
|
|
# Templates read different reasoning keys (reasoning_content: most; reasoning:
|
|
# gemma4; thinking: gpt-oss) — accept any, emit both.
|
|
reasoning = m.get("reasoning_content") or m.get("reasoning") or m.get("thinking")
|
|
if reasoning:
|
|
m.setdefault("reasoning_content", reasoning)
|
|
# gpt-oss's template raises when a tool-call assistant turn carries BOTH content and
|
|
# thinking -- it renders one or the other. Visible text wins: dropping it would lose
|
|
# what the user saw, and every other family reads `content` too.
|
|
if not (m.get("tool_calls") and m.get("content")):
|
|
m.setdefault("thinking", reasoning)
|
|
if m.get("role") == "assistant" and m.get("content") is None:
|
|
# gpt-oss templates concatenate message.content unconditionally.
|
|
m["content"] = ""
|
|
tool_calls = m.get("tool_calls")
|
|
if tool_calls:
|
|
rendered = []
|
|
for tc in tool_calls:
|
|
tc = dict(tc)
|
|
fn = dict(tc.get("function") or {})
|
|
arguments = fn.get("arguments")
|
|
if isinstance(arguments, str):
|
|
try:
|
|
fn["arguments"] = json.loads(arguments)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
tc["function"] = fn
|
|
rendered.append(tc)
|
|
m["tool_calls"] = rendered
|
|
return m
|
|
|
|
|
|
def _flatten_text_parts(parts: list[Any]) -> str | list[dict[str, Any]]:
|
|
"""Plain string for text-only parts; a template-ready part list with {"type": "image"} entries (source kept under freetoken_ref) when images are present."""
|
|
out: list[dict[str, Any]] = []
|
|
has_image = False
|
|
for part in parts:
|
|
ptype = part.get("type") if isinstance(part, dict) else None
|
|
if ptype == "text":
|
|
out.append({"type": "text", "text": part.get("text") or ""})
|
|
elif ptype in ("image_url", "input_image"):
|
|
url = part.get("image_url")
|
|
if isinstance(url, dict):
|
|
url = url.get("url")
|
|
url = url or part.get("url")
|
|
if not url:
|
|
raise ValueError("image content part carries no url")
|
|
out.append({"type": "image", "freetoken_ref": {"kind": "url", "data": url}})
|
|
has_image = True
|
|
elif ptype == "image" and isinstance(part.get("freetoken_ref"), dict):
|
|
out.append(part)
|
|
has_image = True
|
|
else:
|
|
raise ValueError(f"Unsupported content part type: {ptype}")
|
|
if not has_image:
|
|
return "".join(p["text"] for p in out)
|
|
return out
|
|
|
|
|
|
async def _resolve_images(refs: list[dict[str, Any]], state: Any) -> list[bytes]:
|
|
"""Gate and fetch a request's images; every failure is a GenerationError."""
|
|
if reason := image_reject_reason(state.config):
|
|
raise GenerationError(reason)
|
|
try:
|
|
return await fetch_image_bytes(refs, state.config)
|
|
except ValueError as exc:
|
|
raise GenerationError(str(exc)) from exc
|
|
|
|
|
|
def split_tool_lists(
|
|
all_tool_dicts: list[dict[str, Any]] | None, selected_name: str | None = None
|
|
) -> tuple[list[dict[str, Any]] | None, list[dict[str, Any]] | None]:
|
|
"""(template_tools, parser_tools): parser sees all tools; the template sees only
|
|
the selected one when tool_choice forces a specific function. Shared by adapters."""
|
|
if not all_tool_dicts:
|
|
return None, None
|
|
if selected_name:
|
|
template = [t for t in all_tool_dicts if (t.get("function") or {}).get("name") == selected_name]
|
|
else:
|
|
template = all_tool_dicts
|
|
return template, all_tool_dicts
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# The primitive: submit + generate (consume a GenSpec, drive the engine waist).
|
|
# --------------------------------------------------------------------------- #
|
|
async def submit_generation(spec: GenSpec, state: Any) -> int:
|
|
"""Enqueue one generation from a GenSpec; return its uid. Every protocol adapter
|
|
calls this — it takes the neutral spec, not a wire request type."""
|
|
refs = collect_image_refs(spec.messages)
|
|
images = await _resolve_images(refs, state) if refs else None
|
|
uid = state.new_user()
|
|
await state.send_one(
|
|
TokenizeMsg(
|
|
uid=uid,
|
|
text=spec.messages,
|
|
sampling_params=spec.sampling_params,
|
|
chat_template_kwargs=spec.chat_template_kwargs,
|
|
tools=spec.template_tools,
|
|
images=images,
|
|
inline_system_policy=spec.inline_system_policy,
|
|
)
|
|
)
|
|
return uid
|
|
|
|
|
|
async def count_prompt_tokens(
|
|
messages: list[dict[str, Any]],
|
|
tools: list[dict[str, Any]] | None,
|
|
chat_template_kwargs: dict[str, Any],
|
|
state: Any,
|
|
*,
|
|
inline_system_policy: str | None = None,
|
|
) -> int:
|
|
"""Token count of an already-converted (messages, tools, chat_template_kwargs) prompt,
|
|
using the frontend's own tokenizer (``state.frontend_tokenizer()``) so the count equals the
|
|
``usage.input_tokens`` a real generation of the same prompt would report. The neutral
|
|
counterpart to ``submit_generation`` — any protocol's count endpoint converts to this triple
|
|
and calls it. The caller validates the prompt first (non-empty, has tokenizable content).
|
|
|
|
Failure classification mirrors ``/v1/messages``: a chat template that rejects the specific
|
|
conversation (bad role ordering, an unmatched tool_result, an explicit raise_exception) is
|
|
re-raised as ``GenerationError`` — an input-driven, client-classifiable failure, the same
|
|
class the generation path maps to a 400. A tokenizer *initialization* failure (missing
|
|
template, load error) propagates as its original exception, a server fault. Load + tokenize
|
|
run in a worker thread so the event loop is never blocked."""
|
|
refs = collect_image_refs(messages)
|
|
images = await _resolve_images(refs, state) if refs else None
|
|
msg = TokenizeMsg(
|
|
uid=0,
|
|
text=messages,
|
|
sampling_params=SamplingParams(),
|
|
chat_template_kwargs=chat_template_kwargs,
|
|
tools=tools,
|
|
images=images,
|
|
inline_system_policy=inline_system_policy,
|
|
)
|
|
manager = await asyncio.to_thread(state.frontend_tokenizer) # init failure -> server fault
|
|
try:
|
|
(user_msg,) = await asyncio.to_thread(manager.tokenize, [msg])
|
|
except InlineSystemError as exc:
|
|
raise GenerationError(str(exc)) from exc
|
|
except _TemplateError as exc:
|
|
raise GenerationError(str(exc)) from exc
|
|
return int(user_msg.input_ids.numel())
|
|
|
|
|
|
async def prerender_error(spec: GenSpec, state: Any) -> GenerationError | None:
|
|
"""Render ``spec``'s prompt frontend-side, returning the failure a streaming
|
|
adapter should surface as an HTTP 400 *before* committing an SSE stream —
|
|
once headers go out, a template rejection can only ride in-stream, where
|
|
some agents show nothing but "empty response". Render only; the worker
|
|
still renders and encodes authoritatively. Best-effort: a state without a
|
|
frontend tokenizer, or one that fails to *initialize*, skips validation
|
|
rather than blocking the generation path.
|
|
"""
|
|
build = getattr(state, "frontend_tokenizer", None)
|
|
if build is None:
|
|
return None
|
|
msg = TokenizeMsg(
|
|
uid=0,
|
|
text=spec.messages,
|
|
sampling_params=SamplingParams(),
|
|
chat_template_kwargs=spec.chat_template_kwargs,
|
|
tools=spec.template_tools,
|
|
inline_system_policy=spec.inline_system_policy,
|
|
)
|
|
try:
|
|
manager = await asyncio.to_thread(build)
|
|
except Exception: # noqa: BLE001 -- server fault, not this request's problem
|
|
return None
|
|
try:
|
|
await asyncio.to_thread(manager.render_prompt, msg)
|
|
except Exception as exc: # noqa: BLE001 -- mirror the worker's classification
|
|
return GenerationError(f"could not encode request: {exc}")
|
|
return None
|
|
|
|
|
|
def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:
|
|
"""Build a reasoning parser for this generation, or None if the server has no
|
|
reasoning parser configured. ``force_reasoning`` matches the encode-side
|
|
thinking mode so chat-mode content is never mislabeled as reasoning."""
|
|
parser_name = getattr(state.config, "reasoning_parser", None)
|
|
if parser_name == "qwen3":
|
|
# The qwen3 chat template opens an implicit <think> (thinking on) unless
|
|
# enable_thinking is explicitly false, so the model emits only the closing
|
|
# </think>. Mirror that default here, else the chain-of-thought leaks into content.
|
|
force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False
|
|
elif parser_name == "glm":
|
|
# GLM's template honors enable_thinking (default on) even with tools; the
|
|
# generic fallback would force thinking and mislabel disabled output as reasoning.
|
|
force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False
|
|
elif parser_name == "gemma4":
|
|
# Gemma4 defaults thinking off even when tools are present: its template injects an
|
|
# empty thought channel before generation. Do not let Codex tool definitions make all
|
|
# visible text look like hidden reasoning.
|
|
ctk = spec.chat_template_kwargs or {}
|
|
force_reasoning = (
|
|
ctk.get("thinking_mode") == "thinking"
|
|
or bool(ctk.get("enable_thinking"))
|
|
or bool(ctk.get("thinking"))
|
|
)
|
|
elif parser_name == "flint":
|
|
# Flint opens its own optional think block, including when tools are present.
|
|
force_reasoning = False
|
|
elif parser_name == "minimax_m3":
|
|
# M3's template pre-opens <mm:think> only in thinking_mode "enabled" (the
|
|
# model then emits just the closing tag); "adaptive" (default) leaves the
|
|
# model to open the tag itself and "disabled" pre-closes it.
|
|
force_reasoning = (spec.chat_template_kwargs or {}).get("thinking_mode") == "enabled"
|
|
else:
|
|
force_reasoning = (
|
|
resolve_thinking_mode(spec.chat_template_kwargs, spec.template_tools) == "thinking"
|
|
)
|
|
return build_reasoning_parser(state.config, force_reasoning)
|
|
|
|
|
|
def _split_reasoning(text: str, spec: GenSpec, state: Any) -> tuple[str, str]:
|
|
"""Return ``(reasoning, content)``. A no-op (``("", text)``) when no reasoning
|
|
parser is configured, preserving the original behavior for other models."""
|
|
parser = _make_reasoning_parser(spec, state)
|
|
if parser is None:
|
|
return "", text
|
|
return parser.parse_non_stream(text)
|
|
|
|
|
|
def _leaked_special_tokens(state: Any) -> list[str]:
|
|
"""Special-token strings to strip from output. Empty (no-op) unless the dsv4
|
|
reasoning parser is configured, so non-dsv4 output is untouched."""
|
|
return DSV4_SPECIAL_TOKENS if getattr(state.config, "reasoning_parser", None) == "deepseekv32" else []
|
|
|
|
|
|
def _make_tool_parser(spec: GenSpec, state: Any) -> FunctionCallParser:
|
|
"""Build the tool-call parser with its turn-start state read from the prompt:
|
|
a muse detector receives the raw turn bytes (opened by the template's
|
|
``<|start|>assistant``) only when its reasoning parser is not stacked above,
|
|
which otherwise delivers tool slices with full headers."""
|
|
return FunctionCallParser(
|
|
spec.parser_tools or [],
|
|
getattr(state.config, "tool_call_parser", "llama3"),
|
|
turn_starts_open=getattr(state.config, "reasoning_parser", None) != "muse_glimmer",
|
|
)
|
|
|
|
|
|
def _parse_tool_response(
|
|
text: str,
|
|
spec: GenSpec,
|
|
state: Any,
|
|
) -> tuple[str, list[ToolCallItem]] | None:
|
|
if not spec.parse_tools:
|
|
return None
|
|
if not any(tag in text for tag in TOOLS_TAG_LIST):
|
|
return None
|
|
parser = _make_tool_parser(spec, state)
|
|
result = parser.parse_non_stream(text)
|
|
if not result.calls:
|
|
return None
|
|
return result.normal_text, result.calls
|
|
|
|
|
|
def _valid_json(text: str) -> bool:
|
|
if not text:
|
|
return False
|
|
try:
|
|
json.loads(text)
|
|
return True
|
|
except (json.JSONDecodeError, ValueError):
|
|
return False
|
|
|
|
|
|
KEEPALIVE = object()
|
|
"""Sentinel yielded by with_keepalive() when the event stream has been silent."""
|
|
|
|
|
|
async def with_keepalive(events: AsyncIterator[GenEvent], interval: float):
|
|
"""Yield events from ``events``, interspersing the KEEPALIVE sentinel whenever
|
|
``interval`` seconds pass without one (covers queue/prefill silence before the
|
|
first event too). Exceptions propagate unchanged; the pending read is cancelled
|
|
when the consumer closes."""
|
|
aiter = events.__aiter__()
|
|
task = None
|
|
try:
|
|
while True:
|
|
if task is None:
|
|
task = asyncio.ensure_future(aiter.__anext__())
|
|
try:
|
|
ev = await asyncio.wait_for(asyncio.shield(task), interval)
|
|
except asyncio.TimeoutError:
|
|
yield KEEPALIVE
|
|
continue
|
|
except StopAsyncIteration:
|
|
return
|
|
task = None
|
|
yield ev
|
|
finally:
|
|
if task is not None:
|
|
task.cancel()
|
|
|
|
|
|
def _record_generation(
|
|
*,
|
|
source: str | None,
|
|
stream: bool,
|
|
start: float,
|
|
prompt_tokens: int,
|
|
completion_tokens: int,
|
|
error: str | None,
|
|
first_token_at: float | None = None,
|
|
) -> None:
|
|
"""Log one generation request into the request ring. Every protocol adapter converges here,
|
|
so token totals are captured whatever endpoint served the request — unlike the HTTP
|
|
middleware, which for a stream records before the totals are known. `source is None` opts
|
|
out (those paths stay logged by the middleware)."""
|
|
if source is None:
|
|
return
|
|
from .api_server import _served_model_name # lazy: api_server imports this module
|
|
|
|
request_ring.record_request(
|
|
request_ring.RequestRecord(
|
|
ts=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
method="POST",
|
|
path=source,
|
|
status=500 if error else 200,
|
|
model=_served_model_name(),
|
|
duration_ms=int((time.monotonic() - start) * 1000),
|
|
ttft_ms=int((first_token_at - start) * 1000) if first_token_at is not None else None,
|
|
prompt_tokens=prompt_tokens,
|
|
completion_tokens=completion_tokens,
|
|
stream=stream,
|
|
error=error,
|
|
)
|
|
)
|
|
|
|
|
|
async def generate_events(
|
|
uid: int, spec: GenSpec, state: Any, *, source: str | None = None
|
|
) -> AsyncIterator[GenEvent]:
|
|
"""Wraps `_generate_events_impl` to log the request with its totals, read off the terminal
|
|
`GenDone`. The `finally` still records the row on a mid-stream disconnect — but with 0 tokens
|
|
if the drop lands before `GenDone`, the only event carrying the totals."""
|
|
start = time.monotonic()
|
|
prompt_tokens = 0
|
|
completion_tokens = 0
|
|
first_token_at: float | None = None
|
|
error: str | None = None
|
|
try:
|
|
async for ev in _generate_events_impl(uid, spec, state):
|
|
if isinstance(ev, GenDone):
|
|
prompt_tokens = ev.prompt_tokens
|
|
completion_tokens = ev.completion_tokens
|
|
elif first_token_at is None:
|
|
first_token_at = time.monotonic()
|
|
yield ev
|
|
except GenerationError as exc:
|
|
error = str(exc)
|
|
raise
|
|
finally:
|
|
_record_generation(
|
|
source=source, stream=True, start=start,
|
|
prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, error=error,
|
|
first_token_at=first_token_at,
|
|
)
|
|
|
|
|
|
async def generate_full(
|
|
uid: int, spec: GenSpec, state: Any, *, source: str | None = None
|
|
) -> GenResult:
|
|
"""Wraps `_generate_full_impl` to log the request with its totals; the `finally` also records
|
|
a `GenerationError` as a failed row."""
|
|
start = time.monotonic()
|
|
result: GenResult | None = None
|
|
error: str | None = None
|
|
try:
|
|
result = await _generate_full_impl(uid, spec, state)
|
|
return result
|
|
except GenerationError as exc:
|
|
error = str(exc)
|
|
raise
|
|
finally:
|
|
_record_generation(
|
|
source=source, stream=False, start=start,
|
|
prompt_tokens=result.prompt_tokens if result else 0,
|
|
completion_tokens=result.completion_tokens if result else 0,
|
|
error=error,
|
|
)
|
|
|
|
|
|
async def _generate_events_impl(uid: int, spec: GenSpec, state: Any) -> AsyncIterator[GenEvent]:
|
|
"""Protocol-neutral streaming generation. Yields semantic events (reasoning /
|
|
content / tool-call deltas) terminated by exactly one GenDone. Produces no wire
|
|
format — the OpenAI/Anthropic/Responses streamers format these into their own.
|
|
|
|
With tools configured, content is parsed incrementally: plain text streams live
|
|
(the detector holds back only a suffix that could still grow into a tool-call
|
|
tag) and each tool call is emitted as one complete ToolCallsDelta as soon as it
|
|
closes — mid-generation, not after the whole response. Emitted ToolCallItems
|
|
carry the output ordinal (0, 1, ...) in tool_index. Formats whose detector
|
|
cannot parse incrementally (supports_streaming=False) keep the previous
|
|
buffer-everything-then-parse-at-the-end behavior."""
|
|
prompt_tokens = 0
|
|
completion_tokens = 0
|
|
cached_tokens = 0
|
|
pending = ""
|
|
parse_tools = spec.parse_tools
|
|
reasoning_parser = _make_reasoning_parser(spec, state)
|
|
specials = _leaked_special_tokens(state)
|
|
|
|
tool_parser: FunctionCallParser | None = None
|
|
if parse_tools:
|
|
try:
|
|
candidate = _make_tool_parser(spec, state)
|
|
except ValueError:
|
|
candidate = None # unsupported parser name: keep the buffered path's behavior
|
|
if candidate is not None and candidate.supports_streaming():
|
|
tool_parser = candidate
|
|
frag_stable = tool_parser.args_fragments_prefix_stable() if tool_parser else True
|
|
|
|
# Streaming tool-call assembly: detectors emit fragments (name first, then
|
|
# argument diffs); they accumulate here and the call is emitted complete when
|
|
# the next call starts, trailing text arrives, or the stream ends.
|
|
open_call: dict[str, Any] | None = None
|
|
calls_emitted = 0
|
|
# Swallow whitespace-only text only between a call's close and the next real
|
|
# text (markup separators) — NOT inside post-call prose, where a lone " "
|
|
# chunk is a legitimate word gap.
|
|
suppress_ws = False
|
|
|
|
def _close_open_call() -> ToolCallsDelta | None:
|
|
nonlocal open_call, calls_emitted
|
|
if open_call is None:
|
|
return None
|
|
# Streamed fragments concatenate to the exact final arguments (detectors
|
|
# emit prefix-stable fragments and close the JSON before the call ends);
|
|
# fall back to the detector's parse state when the stream was cut short.
|
|
params = open_call["params"]
|
|
if not _valid_json(params):
|
|
params = tool_parser.unstreamed_arguments(open_call["detector_index"]) or params
|
|
call = ToolCallItem(
|
|
tool_index=open_call["ordinal"], name=open_call["name"], parameters=params or "{}"
|
|
)
|
|
open_call = None
|
|
calls_emitted += 1
|
|
nonlocal suppress_ws
|
|
suppress_ws = True
|
|
return ToolCallsDelta([call])
|
|
|
|
def _route_tool_text(piece: str) -> list[GenEvent]:
|
|
nonlocal open_call, suppress_ws
|
|
out: list[GenEvent] = []
|
|
if not piece:
|
|
return out
|
|
for kind, payload in tool_parser.parse_stream_events(piece):
|
|
if kind == "text":
|
|
# Text arriving while a call is open means the call finished:
|
|
# close it first so wire order matches generation order.
|
|
done = _close_open_call()
|
|
if done is not None:
|
|
out.append(done)
|
|
stripped = strip_special_tokens(payload, specials)
|
|
if stripped and not (stripped.strip() == "" and suppress_ws):
|
|
out.append(ContentDelta(stripped))
|
|
if stripped.strip():
|
|
suppress_ws = False
|
|
continue
|
|
for frag in payload:
|
|
starts_new = frag.name is not None and (
|
|
open_call is None or frag.tool_index != open_call["detector_index"]
|
|
)
|
|
if starts_new:
|
|
done = _close_open_call()
|
|
if done is not None:
|
|
out.append(done)
|
|
if open_call is None or starts_new:
|
|
open_call = {
|
|
"detector_index": frag.tool_index,
|
|
"name": frag.name,
|
|
"params": "",
|
|
"ordinal": calls_emitted,
|
|
}
|
|
out.append(ToolCallStart(
|
|
tool_index=calls_emitted, name=frag.name, args_prefix_stable=frag_stable,
|
|
))
|
|
if frag.parameters:
|
|
open_call["params"] += frag.parameters
|
|
out.append(
|
|
ToolCallArgsDelta(tool_index=open_call["ordinal"], fragment=frag.parameters)
|
|
)
|
|
return out
|
|
|
|
engine_finish_reason: str | None = None
|
|
engine_matched_stop: str | None = None
|
|
timings: dict[str, Any] | None = None
|
|
async for ack in state.wait_for_ack(uid):
|
|
if getattr(ack, "error", None):
|
|
raise GenerationError(ack.error, getattr(ack, "error_code", None))
|
|
prompt_tokens += ack.prompt_tokens_delta
|
|
completion_tokens += ack.completion_tokens_delta
|
|
cached_tokens += ack.cached_tokens
|
|
if getattr(ack, "timings", None) is not None:
|
|
timings = ack.timings
|
|
content_delta = ack.incremental_output
|
|
if reasoning_parser is not None and content_delta:
|
|
reasoning_delta, content_delta = reasoning_parser.parse_stream_chunk(content_delta)
|
|
if reasoning_delta:
|
|
stripped_reasoning = strip_special_tokens(reasoning_delta, specials)
|
|
if stripped_reasoning: # a bare special token must not open a thinking block
|
|
yield ReasoningDelta(stripped_reasoning)
|
|
if content_delta:
|
|
if tool_parser is not None:
|
|
for ev in _route_tool_text(content_delta):
|
|
yield ev
|
|
elif parse_tools:
|
|
pending += content_delta
|
|
else:
|
|
yield ContentDelta(strip_special_tokens(content_delta, specials))
|
|
if ack.finished:
|
|
engine_finish_reason = getattr(ack, "finish_reason", None)
|
|
engine_matched_stop = getattr(ack, "matched_stop", None)
|
|
break
|
|
|
|
# Drain residue held in the reasoning parser (a deferred tool block, or a
|
|
# trailing partial token) so it is not silently dropped.
|
|
if reasoning_parser is not None:
|
|
flush_reasoning, flush_content = reasoning_parser.flush()
|
|
if flush_reasoning:
|
|
stripped_reasoning = strip_special_tokens(flush_reasoning, specials)
|
|
if stripped_reasoning:
|
|
yield ReasoningDelta(stripped_reasoning)
|
|
if flush_content:
|
|
if tool_parser is not None:
|
|
for ev in _route_tool_text(flush_content):
|
|
yield ev
|
|
elif parse_tools:
|
|
pending += flush_content
|
|
else:
|
|
yield ContentDelta(strip_special_tokens(flush_content, specials))
|
|
|
|
# Engine reason ("stop"/"length"); a tool call overrides it, but a truncation (length) wins.
|
|
finish_reason = engine_finish_reason or "stop"
|
|
|
|
if tool_parser is not None:
|
|
# End-of-stream drain: let the detector finalize a call cut off mid-arguments
|
|
# (closing fragments keep the client's concatenated JSON valid), close a call
|
|
# whose end marker never arrived (truncated generation), best-effort recover a
|
|
# call cut off inside an unterminated tag block, then release text still held
|
|
# back for tag disambiguation.
|
|
for frag in tool_parser.finalize_stream():
|
|
if open_call is not None and frag.parameters:
|
|
open_call["params"] += frag.parameters
|
|
yield ToolCallArgsDelta(tool_index=open_call["ordinal"], fragment=frag.parameters)
|
|
done = _close_open_call()
|
|
if done is not None:
|
|
yield done
|
|
for item in tool_parser.recover_truncated_call():
|
|
yield ToolCallStart(tool_index=calls_emitted, name=item.name)
|
|
yield ToolCallsDelta(
|
|
[ToolCallItem(tool_index=calls_emitted, name=item.name, parameters=item.parameters)]
|
|
)
|
|
calls_emitted += 1
|
|
residual = tool_parser.finish_stream()
|
|
if residual:
|
|
stripped = strip_special_tokens(residual, specials)
|
|
if stripped and not (stripped.strip() == "" and suppress_ws):
|
|
yield ContentDelta(stripped)
|
|
if calls_emitted and finish_reason != "length":
|
|
finish_reason = "tool_calls"
|
|
else:
|
|
parsed = _parse_tool_response(pending, spec, state) if parse_tools else None
|
|
if parsed is not None:
|
|
normal_text, tool_calls = parsed
|
|
normal_text = strip_special_tokens(normal_text, specials)
|
|
if normal_text:
|
|
yield ContentDelta(normal_text)
|
|
yield ToolCallsDelta(tool_calls)
|
|
if finish_reason != "length":
|
|
finish_reason = "tool_calls"
|
|
elif parse_tools and pending:
|
|
yield ContentDelta(strip_special_tokens(pending, specials))
|
|
|
|
yield GenDone(
|
|
finish_reason, prompt_tokens, completion_tokens,
|
|
matched_stop=engine_matched_stop, cached_tokens=cached_tokens,
|
|
timings=timings,
|
|
)
|
|
|
|
|
|
async def _generate_full_impl(uid: int, spec: GenSpec, state: Any) -> GenResult:
|
|
"""Protocol-neutral non-streaming generation: accumulate, split reasoning, parse
|
|
tool calls, strip special tokens. The adapters format the GenResult into their wire."""
|
|
full_content = ""
|
|
prompt_tokens = 0
|
|
completion_tokens = 0
|
|
cached_tokens = 0
|
|
timings: dict[str, Any] | None = None
|
|
engine_finish_reason: str | None = None
|
|
engine_matched_stop: str | None = None
|
|
async for ack in state.wait_for_ack(uid):
|
|
if getattr(ack, "error", None):
|
|
raise GenerationError(ack.error, getattr(ack, "error_code", None))
|
|
prompt_tokens += ack.prompt_tokens_delta
|
|
completion_tokens += ack.completion_tokens_delta
|
|
cached_tokens += ack.cached_tokens
|
|
if getattr(ack, "timings", None) is not None:
|
|
timings = ack.timings
|
|
full_content += ack.incremental_output
|
|
if ack.finished:
|
|
engine_finish_reason = getattr(ack, "finish_reason", None)
|
|
engine_matched_stop = getattr(ack, "matched_stop", None)
|
|
break
|
|
|
|
reasoning_text, content_text = _split_reasoning(full_content, spec, state)
|
|
# Engine reason ("stop"/"length"); a tool call overrides it, but a truncation (length) wins.
|
|
finish_reason = engine_finish_reason or "stop"
|
|
tool_calls: list[ToolCallItem] = []
|
|
parsed = _parse_tool_response(content_text, spec, state)
|
|
if parsed is not None:
|
|
content_text, tool_calls = parsed
|
|
if finish_reason != "length":
|
|
finish_reason = "tool_calls"
|
|
|
|
specials = _leaked_special_tokens(state)
|
|
return GenResult(
|
|
reasoning=strip_special_tokens(reasoning_text, specials).strip(),
|
|
content=strip_special_tokens(content_text, specials),
|
|
tool_calls=tool_calls,
|
|
finish_reason=finish_reason,
|
|
prompt_tokens=prompt_tokens,
|
|
completion_tokens=completion_tokens,
|
|
matched_stop=engine_matched_stop,
|
|
cached_tokens=cached_tokens,
|
|
timings=timings,
|
|
)
|