Files
Xiaoze Fan e328cf6381 refactor: add engine and scheduler seams for multi-row decode steps (#833)
* refactor: add engine and scheduler seams for multi-row decode steps

* refactor: rename the graph steps argument to rows_per_req

* refactor: drop VerifyInfo and require rows_per_req on verify batches

* refactor: fix stale seam comments and read decode_rows_per_req directly

* refactor: reword two GraphRunner comments for multi-row graphs

* fix(engine): scale the GPU-decode MoE pool limits by decode_rows_per_req

* refactor: drop the unread Context.moe_offload_cache field

* test: trim the multi-row seam tests
2026-10-27 16:03:01 -07:00

150 lines
6.2 KiB
Python

from dataclasses import dataclass
from typing import Dict, FrozenSet, List
from freetoken.message import DetokenizeMsg
from transformers import PreTrainedTokenizerBase
# Borrowed from sglang
def _is_chinese_char(cp: int):
"""Checks whether CP is the codepoint of a CJK character."""
# This defines a "chinese character" as anything in the CJK Unicode block:
# https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
#
# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
# despite its name. The modern Korean Hangul alphabet is a different block,
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
# space-separated words, so they are not treated specially and handled
# like the all of the other languages.
if (
(cp >= 0x4E00 and cp <= 0x9FFF)
or (cp >= 0x3400 and cp <= 0x4DBF) #
or (cp >= 0x20000 and cp <= 0x2A6DF) #
or (cp >= 0x2A700 and cp <= 0x2B73F) #
or (cp >= 0x2B740 and cp <= 0x2B81F) #
or (cp >= 0x2B820 and cp <= 0x2CEAF) #
or (cp >= 0xF900 and cp <= 0xFAFF)
or (cp >= 0x2F800 and cp <= 0x2FA1F) #
): #
return True
return False
def find_printable_text(text: str):
"""Returns the longest printable substring of text that contains only entire words."""
# Borrowed from https://github.com/huggingface/transformers/blob/061580c82c2db1de9139528243e105953793f7a2/src/transformers/generation/streamers.py#L99
# After the symbol for a new line, we flush the cache.
if text.endswith("\n"):
return text
# If the last token is a CJK character, we print the characters.
elif len(text) > 0 and _is_chinese_char(ord(text[-1])):
return text
# Otherwise if the penultimate token is a CJK character, we print the characters except for the last one.
elif len(text) > 1 and _is_chinese_char(ord(text[-2])):
return text[:-1]
# Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
# which may change with the subsequent token -- there are probably smarter ways to do this!)
else:
return text[: text.rfind(" ") + 1]
def _stop_prefix_holdback(text: str, stop_strs: list[str]) -> int:
"""Length of the longest trailing suffix of ``text`` that is a proper prefix of some
stop string. Those chars are withheld until a later token resolves whether the stop
completes, so a partial stop is never streamed and then needs retracting."""
hold = 0
for stop in stop_strs:
for i in range(min(len(stop) - 1, len(text)), 0, -1):
if text.endswith(stop[:i]):
hold = max(hold, i)
break
return hold
@dataclass
class DecodeStatus:
decoded_ids: List[int]
decoded_str: str
read_offset: int # length of read ids
surr_offset: int # length of surr ids
sent_offset: int # length of sent out string
class DetokenizeManager:
def __init__(
self, tokenizer: PreTrainedTokenizerBase, eos_token_ids: FrozenSet[int] | None = None
) -> None:
# uid -> DecodeStatus
self.decode_map: Dict[int, DecodeStatus] = {}
self.tokenizer = tokenizer
self.eos_token_ids = (
eos_token_ids if eos_token_ids is not None else frozenset({tokenizer.eos_token_id})
)
def discard(self, uid: int) -> None:
"""Drop a uid's decode state without a finished DetokenizeMsg. An aborted or errored
request never sends one (its terminal reply is an ErrorReplyMsg), so without this its
accumulated ids and text stay in ``decode_map`` for the life of the worker."""
self.decode_map.pop(uid, None)
def detokenize(self, msgs: List[DetokenizeMsg]) -> List[str]:
read_ids: List[List[int]] = []
surr_ids: List[List[int]] = []
for msg in msgs:
if msg.uid not in self.decode_map:
self.decode_map[msg.uid] = DecodeStatus(
decoded_ids=[],
decoded_str="",
read_offset=0,
surr_offset=0,
sent_offset=0,
)
s = self.decode_map[msg.uid]
tokens = msg.next_tokens
if msg.finished and tokens and tokens[-1] in self.eos_token_ids:
tokens = tokens[:-1]
s.decoded_ids.extend(tokens)
read_ids.append(s.decoded_ids[s.surr_offset :])
surr_ids.append(s.decoded_ids[s.surr_offset : s.read_offset])
read_texts = self.tokenizer.batch_decode(read_ids)
surr_texts = self.tokenizer.batch_decode(surr_ids)
incremental_strs: List[str] = []
for msg, read_str, surr_str in zip(msgs, read_texts, surr_texts, strict=True):
s = self.decode_map[msg.uid]
new_text = read_str[len(surr_str) :]
# Streaming chunk: update the decode status
if len(new_text) > 0 and not new_text.endswith("�"):
output_str = s.decoded_str + new_text
s.decoded_str = output_str
s.surr_offset = s.read_offset
s.read_offset = len(s.decoded_ids)
else:
new_text = find_printable_text(new_text)
output_str = s.decoded_str + new_text
prev_sent = s.sent_offset
if msg.finished:
# Generation is over: flush everything, trimming at the matched stop string.
if msg.matched_stop:
cut = output_str.find(msg.matched_stop)
emit_end = cut if cut >= 0 else len(output_str)
else:
emit_end = len(output_str)
elif msg.stop_strs:
# Hold back a trailing suffix that could still grow into a stop string.
emit_end = len(output_str) - _stop_prefix_holdback(output_str, msg.stop_strs)
else:
emit_end = len(output_str)
incremental_output = output_str[prev_sent:emit_end] if emit_end > prev_sent else ""
s.sent_offset = max(prev_sent, emit_end)
incremental_strs.append(incremental_output)
if msg.finished:
del self.decode_map[msg.uid]
return incremental_strs