mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
335 lines
15 KiB
Python
335 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import threading
|
|
from copy import deepcopy
|
|
from types import ModuleType
|
|
from typing import TYPE_CHECKING, Any, List
|
|
|
|
import torch
|
|
from freetoken.message import TokenizeMsg, UserMsg
|
|
from freetoken.utils import init_logger
|
|
from transformers import PreTrainedTokenizerBase
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.mm.processor import MMProcessor
|
|
|
|
from .effort import (
|
|
EffortProfile,
|
|
ThinkingProfile,
|
|
probe_effort_profile,
|
|
probe_thinking_profile,
|
|
quantize_effort,
|
|
)
|
|
from .inline_system import has_inline_system, normalize_inline_system, probe_inline_system
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def resolve_thinking_mode(chat_template_kwargs: dict[str, Any] | None, tools: Any | None) -> str:
|
|
"""Resolve the thinking mode (``"thinking"`` or ``"chat"``) for a chat request.
|
|
|
|
The single source of truth for this decision: the encode side
|
|
(``_apply_dsv4_chat_encoder`` below) uses it to pick the prompt the model
|
|
sees, and the frontend parse side (``server/openai_api.py``) imports it to
|
|
decide whether the model's output begins inside a reasoning block. Keeping
|
|
one implementation prevents the two sides from disagreeing. Thinking is on
|
|
when tools are offered (dsv4 only emits well-formed tool calls in thinking
|
|
mode) or when the caller requests it via ``chat_template_kwargs``.
|
|
"""
|
|
ctk = chat_template_kwargs or {}
|
|
mode = str(ctk.get("thinking_mode") or "chat")
|
|
if tools or ctk.get("enable_thinking") or ctk.get("thinking"):
|
|
mode = "thinking"
|
|
if mode not in ("chat", "thinking"):
|
|
mode = "chat"
|
|
return mode
|
|
|
|
|
|
_EFFORT_PROBE_MESSAGES = [{"role": "user", "content": "ping"}]
|
|
|
|
|
|
class TokenizeManager:
|
|
def __init__(self, tokenizer: PreTrainedTokenizerBase, mm_processor: MMProcessor | None = None) -> None:
|
|
self.tokenizer = tokenizer
|
|
self.mm_processor = mm_processor # None: the model takes no images
|
|
self._dsv4_encoder = _load_dsv4_encoder_if_needed(tokenizer)
|
|
self._effort_profile: EffortProfile | None = None
|
|
self._thinking_profile: ThinkingProfile | None = None
|
|
self._effort_lock = threading.Lock()
|
|
self._logged_effort_maps: set[tuple[Any, str | int | None]] = set()
|
|
self._inline_system_lock = threading.Lock()
|
|
self._inline_system_modes: dict[str, str] = {}
|
|
|
|
def tokenize(self, msgs: List[TokenizeMsg]) -> List[UserMsg]:
|
|
results: List[UserMsg] = []
|
|
# TODO: batch tokenization
|
|
for msg in msgs:
|
|
prompt = self.render_prompt(msg)
|
|
# A jinja chat template owns every special token (HF's apply_chat_template
|
|
# tokenizes with add_special_tokens=False for the same reason): tokenizers
|
|
# that auto-add bos (muse-glimmer's, llama's) would otherwise double it --
|
|
# the template already rendered one. Raw-string prompts and the dsv4
|
|
# encoder path keep the default.
|
|
templated = isinstance(msg.text, list) and self._dsv4_encoder is None
|
|
input_ids: torch.Tensor = ( # type: ignore
|
|
self.tokenizer.encode(
|
|
prompt, return_tensors="pt", add_special_tokens=not templated
|
|
)
|
|
)
|
|
input_ids = input_ids.view(-1).to(torch.int32)
|
|
if msg.images:
|
|
if self.mm_processor is None:
|
|
raise ValueError("image input is not supported for this model")
|
|
mm = self.mm_processor.apply(input_ids, msg.images)
|
|
results.append(
|
|
UserMsg(
|
|
uid=msg.uid,
|
|
input_ids=mm.input_ids,
|
|
sampling_params=msg.sampling_params,
|
|
mm_items=mm.mm_items,
|
|
mrope_positions=mm.mrope_positions,
|
|
mrope_delta=mm.mrope_delta,
|
|
)
|
|
)
|
|
else:
|
|
results.append(
|
|
UserMsg(uid=msg.uid, input_ids=input_ids, sampling_params=msg.sampling_params)
|
|
)
|
|
return results
|
|
|
|
def render_prompt(self, msg: TokenizeMsg) -> str:
|
|
"""The template/encoder half of ``tokenize``, exposed so the frontend can
|
|
validate a request before committing an SSE stream. Sanitizes
|
|
``reasoning_effort`` first: every render path (worker, frontend
|
|
validation, count_tokens) must quantize identically."""
|
|
if not isinstance(msg.text, list):
|
|
return msg.text
|
|
kwargs = self._sanitize_effort(msg.chat_template_kwargs or {})
|
|
messages = msg.text
|
|
if msg.inline_system_policy is not None:
|
|
if msg.inline_system_policy not in ("auto", "preserve", "fold"):
|
|
raise ValueError(f"invalid inline system policy: {msg.inline_system_policy}")
|
|
if has_inline_system(messages):
|
|
mode = msg.inline_system_policy
|
|
if mode == "auto":
|
|
mode = self.inline_system_mode(msg.tools, kwargs)
|
|
messages = normalize_inline_system(messages, mode)
|
|
return self._render(messages, msg.tools, kwargs, images=msg.images)
|
|
|
|
def inline_system_mode(self, tools: list[dict] | None, kwargs: dict[str, Any]) -> str:
|
|
key = json.dumps([tools, kwargs], sort_keys=True)
|
|
with self._inline_system_lock:
|
|
if key not in self._inline_system_modes:
|
|
mode, reason = probe_inline_system(
|
|
lambda messages: self._render(messages, deepcopy(tools), deepcopy(kwargs)), self.tokenizer
|
|
)
|
|
if len(self._inline_system_modes) >= 128:
|
|
self._inline_system_modes.pop(next(iter(self._inline_system_modes)))
|
|
self._inline_system_modes[key] = mode
|
|
logger.info("Anthropic inline system policy: %s (%s)", mode, reason)
|
|
return self._inline_system_modes[key]
|
|
|
|
def _render(
|
|
self,
|
|
messages: list[dict[str, Any]],
|
|
tools: list[dict[str, Any]] | None,
|
|
chat_template_kwargs: dict[str, Any],
|
|
*, images: list[bytes] | None = None,
|
|
) -> str:
|
|
"""Raw render, no effort sanitation — the probe needs unsupported values
|
|
to actually reach the template so rejection is observable."""
|
|
if self._dsv4_encoder is not None:
|
|
return _apply_dsv4_chat_encoder(
|
|
self._dsv4_encoder, messages, tools, chat_template_kwargs, images=images
|
|
)
|
|
if any(m.get("role") == "developer" for m in messages):
|
|
# OpenAI's "developer" is the newer spelling of "system". Templates that do not
|
|
# name it drop it, raise on it, or render it as a role the model never saw.
|
|
if "developer" not in str(self.tokenizer.chat_template):
|
|
messages = [{**m, "role": "system"} if m.get("role") == "developer" else m for m in messages]
|
|
# Broadcast the effort in every spelling the ecosystem's templates read
|
|
# (muse-glimmer grades ``reasoning_strength``; Jinja ignores undeclared
|
|
# variables) -- the same rule the thinking toggles use. An explicit
|
|
# caller-provided spelling wins over the broadcast.
|
|
if "reasoning_effort" in chat_template_kwargs:
|
|
chat_template_kwargs = dict(chat_template_kwargs)
|
|
chat_template_kwargs.setdefault(
|
|
"reasoning_strength", chat_template_kwargs["reasoning_effort"]
|
|
)
|
|
if tools is not None:
|
|
chat_template_kwargs = {**chat_template_kwargs, "tools": tools}
|
|
prompt = self.tokenizer.apply_chat_template(
|
|
messages,
|
|
tokenize=False,
|
|
add_generation_prompt=True,
|
|
**chat_template_kwargs,
|
|
)
|
|
assert isinstance(prompt, str)
|
|
return prompt
|
|
|
|
def effort_profile(self) -> EffortProfile:
|
|
"""The checkpoint's effort vocabulary, probed on first use and cached
|
|
for the process lifetime."""
|
|
with self._effort_lock:
|
|
if self._effort_profile is None:
|
|
self._effort_profile = probe_effort_profile(self._probe_render)
|
|
logger.info(
|
|
"reasoning-effort profile: supported=%s default=%s",
|
|
sorted(self._effort_profile.supported) or "(none)",
|
|
self._effort_profile.default,
|
|
)
|
|
return self._effort_profile
|
|
|
|
def thinking_profile(self) -> ThinkingProfile:
|
|
"""The checkpoint's thinking controls (toggle behavior + effort
|
|
vocabulary), probed on first use and cached for the process lifetime.
|
|
Feeds the /v1/cache/status gear derivation."""
|
|
efforts = self.effort_profile()
|
|
with self._effort_lock:
|
|
if self._thinking_profile is None:
|
|
self._thinking_profile = probe_thinking_profile(self._probe_render, efforts)
|
|
return self._thinking_profile
|
|
|
|
def _probe_render(
|
|
self, kwargs: dict[str, Any], tools: list[dict[str, Any]] | None
|
|
) -> str:
|
|
return self._render(_EFFORT_PROBE_MESSAGES, tools, kwargs)
|
|
|
|
def _sanitize_effort(self, chat_template_kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
if "reasoning_effort" not in chat_template_kwargs:
|
|
return chat_template_kwargs
|
|
raw = chat_template_kwargs.get("reasoning_effort")
|
|
mapped = quantize_effort(raw, self.effort_profile())
|
|
if mapped == raw:
|
|
return chat_template_kwargs
|
|
# raw is client-controlled and may be unhashable (a JSON list/dict).
|
|
key = (raw if isinstance(raw, str) else repr(raw), mapped)
|
|
if key not in self._logged_effort_maps:
|
|
self._logged_effort_maps.add(key)
|
|
logger.info(
|
|
"reasoning_effort %r is not supported by this checkpoint; using %s",
|
|
raw,
|
|
mapped if mapped is not None else "the template default",
|
|
)
|
|
sanitized = dict(chat_template_kwargs)
|
|
if mapped is None:
|
|
del sanitized["reasoning_effort"]
|
|
else:
|
|
sanitized["reasoning_effort"] = mapped
|
|
return sanitized
|
|
|
|
|
|
def _load_dsv4_encoder_if_needed(tokenizer: PreTrainedTokenizerBase) -> ModuleType | None:
|
|
if getattr(tokenizer, "chat_template", None):
|
|
return None
|
|
model_path = getattr(tokenizer, "name_or_path", None) or getattr(tokenizer, "_name_or_path", "")
|
|
if not model_path:
|
|
return None
|
|
# DeepSeek-V4 ships encoding/encoding_dsv4.py, V4.1 encoding/encoding.py; same encode_messages contract
|
|
encoder_path = next(
|
|
(p for p in (os.path.join(str(model_path), "encoding", name) for name in ("encoding_dsv4.py", "encoding.py")) if os.path.isfile(p)),
|
|
None,
|
|
)
|
|
if encoder_path is None:
|
|
return None
|
|
spec = importlib.util.spec_from_file_location("encoding_dsv4", encoder_path)
|
|
if spec is None or spec.loader is None:
|
|
return None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
if not hasattr(module, "encode_messages"):
|
|
return None
|
|
return module
|
|
|
|
|
|
def _apply_dsv4_chat_encoder(
|
|
encoder: ModuleType,
|
|
messages: list[dict],
|
|
tools: list[dict] | None,
|
|
chat_template_kwargs: dict,
|
|
*, images: list[bytes] | None = None,
|
|
) -> str:
|
|
rendered_messages = [dict(message) for message in messages]
|
|
image_data = iter(images) if images is not None else None
|
|
for message in rendered_messages:
|
|
if isinstance(message.get("content"), list):
|
|
content = []
|
|
for part in message["content"]:
|
|
if isinstance(part, dict) and part.get("type") == "image":
|
|
# Prerender still has refs; the tokenizer worker receives fetched bytes.
|
|
# The checkpoint encoder validates sources even when rendering only text.
|
|
ref = part.get("freetoken_ref")
|
|
if image_data is not None:
|
|
try:
|
|
part = {"type": "image", "data": next(image_data)}
|
|
except StopIteration:
|
|
raise ValueError("image blocks outnumber fetched images") from None
|
|
elif ref is not None:
|
|
key = "data" if ref["kind"] == "b64" else "url"
|
|
part = {"type": "image", key: ref["data"]}
|
|
content.append(part)
|
|
message["content"] = content
|
|
if message.get("tool_calls"):
|
|
message["tool_calls"] = _dsv4_tool_calls(message["tool_calls"])
|
|
if tools:
|
|
_attach_tools_to_dsv4_messages(rendered_messages, tools)
|
|
|
|
# No effort filtering here: the caller sanitized already, and the probe
|
|
# needs raw values to reach the encoder's own validation.
|
|
return encoder.encode_messages(
|
|
rendered_messages,
|
|
thinking_mode=resolve_thinking_mode(chat_template_kwargs, tools),
|
|
reasoning_effort=chat_template_kwargs.get("reasoning_effort"),
|
|
)
|
|
|
|
|
|
def _dsv4_tool_calls(tool_calls: list[dict]) -> list[dict]:
|
|
"""The dsv4 encoder's contract is ``function.arguments`` = JSON-object STRING
|
|
(it json.loads then iterates .items()); a dict (what ``render_messages``
|
|
produces for Jinja templates) trips its bare-except fallback, which wraps the
|
|
whole payload in a bogus parameter literally named ``arguments``. Re-serialize
|
|
here. Copies each tool-call dict: the outer message copy is shallow, so these
|
|
are shared with the caller."""
|
|
rendered = []
|
|
for tc in tool_calls:
|
|
tc = dict(tc)
|
|
fn = dict(tc.get("function") or {})
|
|
fn["arguments"] = _dsv4_arguments_str(fn.get("arguments"))
|
|
tc["function"] = fn
|
|
rendered.append(tc)
|
|
return rendered
|
|
|
|
|
|
def _dsv4_arguments_str(arguments: Any) -> str:
|
|
"""Missing/empty means no arguments (vLLM parity); anything else that is not
|
|
a JSON object is rejected -- ValueError becomes a per-request "could not
|
|
encode request" error, never a worker crash -- matching sglang's
|
|
validate-then-400. A non-object would otherwise raise uncaught in the
|
|
encoder's .items() or be wrapped as garbage."""
|
|
if arguments is None or (isinstance(arguments, str) and not arguments.strip()):
|
|
return "{}"
|
|
if isinstance(arguments, dict):
|
|
return json.dumps(arguments, ensure_ascii=False)
|
|
shown = f"{arguments!r:.200}"
|
|
if isinstance(arguments, str):
|
|
try:
|
|
parsed = json.loads(arguments)
|
|
except json.JSONDecodeError as err:
|
|
raise ValueError(
|
|
f"tool call function.arguments must be valid JSON, got {shown}"
|
|
) from err
|
|
if isinstance(parsed, dict):
|
|
return arguments
|
|
raise ValueError(f"tool call function.arguments must be a JSON object, got {shown}")
|
|
|
|
|
|
def _attach_tools_to_dsv4_messages(messages: list[dict], tools: list[dict]) -> None:
|
|
for message in messages:
|
|
if message.get("role") == "system":
|
|
message["tools"] = tools
|
|
return
|
|
messages.insert(0, {"role": "system", "content": "", "tools": tools})
|