from __future__ import annotations import multiprocessing as mp from typing import Any, List import torch from freetoken.message import ( AbortBackendMsg, AbortMsg, BaseBackendMsg, BaseFrontendMsg, BaseTokenizerMsg, BatchBackendMsg, BatchFrontendMsg, BatchTokenizerMsg, CacheRebuildBackendMsg, CacheRebuildMsg, CacheRebuildReply, CacheRebuildResultMsg, DetokenizeMsg, ErrorReplyMsg, FlushCacheBackendMsg, FlushCacheMsg, FlushCacheReply, FlushCacheResultMsg, PromptAdmittedMsg, TokenizeMsg, UserMsg, UserReply, ) from freetoken.mm.config import MultimodalConfig from freetoken.utils import ( ZmqPullQueue, ZmqPushQueue, init_logger, load_eos_token_ids, load_tokenizer, ) def _unwrap_msg(msg: BaseTokenizerMsg) -> List[BaseTokenizerMsg]: if isinstance(msg, BatchTokenizerMsg): return msg.data return [msg] def _prompt_admitted_reply(msg: PromptAdmittedMsg) -> UserReply: """Translate the scheduler's admission signal onto the existing frontend usage path.""" return UserReply( uid=msg.uid, incremental_output="", finished=False, prompt_tokens_delta=msg.prompt_tokens, cached_tokens=msg.cached_tokens, ) def _error_reply(msg: ErrorReplyMsg) -> UserReply: return UserReply( uid=msg.uid, incremental_output="", finished=True, error=msg.error, error_code=msg.code, ) def _put_user_replies(send_frontend: Any, replies: List[UserReply]) -> None: if replies: send_frontend.put( replies[0] if len(replies) == 1 else BatchFrontendMsg(data=replies) ) def _send_generation_replies( send_frontend: Any, admitted: List[UserReply], sampled: List[UserReply], terminal_errors: List[UserReply], ) -> None: """Preserve the accounting barrier within one tokenizer queue drain. Scheduler abort acknowledgements are terminal: every already-sampled DetokenizeMsg drained alongside them must reach FrontendManager first. Admission messages remain first so per-request usage precedes that request's sampled completion. """ _put_user_replies(send_frontend, admitted) _put_user_replies(send_frontend, sampled) _put_user_replies(send_frontend, terminal_errors) def _tokenize_requests( tokenize_manager: Any, messages: List[TokenizeMsg], logger: Any, ) -> tuple[List[UserMsg], List[UserReply]]: """Tokenize independently, returning backend work plus terminal frontend errors. Successful tokenization deliberately emits no prompt-token reply: accounting starts only when the scheduler later confirms first-prefill admission. """ backend: List[UserMsg] = [] errors: List[UserReply] = [] for msg in messages: try: user_msg = tokenize_manager.tokenize([msg])[0] except Exception as exc: # noqa: BLE001 — isolate, never crash the worker logger.warning(f"tokenization failed for request {msg.uid}: {exc!r}") errors.append( UserReply( uid=msg.uid, incremental_output="", finished=True, error=f"could not encode request: {exc}", ) ) continue # A zero-token prompt would trip the scheduler's input_len > 0 invariant and # crash the worker; reject it here as a terminal error instead. if user_msg.input_ids.numel() == 0: errors.append( UserReply( uid=msg.uid, incremental_output="", finished=True, error="prompt must contain at least one token", ) ) continue backend.append(user_msg) return backend, errors @torch.inference_mode() def tokenize_worker( *, tokenizer_path: str, addr: str, create: bool, backend_addr: str, frontend_addr: str, local_bs: int, tokenizer_id: int = -1, model_source: str = "huggingface", ack_queue: mp.Queue[str] | None = None, mm: MultimodalConfig | None = None, ) -> None: send_backend = ZmqPushQueue(backend_addr, create=False, encoder=BaseBackendMsg.encoder) send_frontend = ZmqPushQueue(frontend_addr, create=False, encoder=BaseFrontendMsg.encoder) recv_listener = ZmqPullQueue(addr, create=create, decoder=BatchTokenizerMsg.decoder) assert local_bs > 0 tokenizer = load_tokenizer(tokenizer_path) logger = init_logger(__name__, f"tokenizer_{tokenizer_id}") from freetoken.mm.processor import get_mm_processor from .detokenize import DetokenizeManager from .tokenize import TokenizeManager tokenize_manager = TokenizeManager(tokenizer, get_mm_processor(tokenizer_path, mm)) detokenize_manager = DetokenizeManager( tokenizer, load_eos_token_ids(tokenizer_path, tokenizer) ) if ack_queue is not None: ack_queue.put(f"Tokenize server {tokenizer_id} is ready") try: while True: pending_msg = _unwrap_msg(recv_listener.get()) while len(pending_msg) < local_bs and not recv_listener.empty(): pending_msg.extend(_unwrap_msg(recv_listener.get())) logger.debug(f"Received {len(pending_msg)} messages") detokenize_msg = [m for m in pending_msg if isinstance(m, DetokenizeMsg)] tokenize_msg = [m for m in pending_msg if isinstance(m, TokenizeMsg)] abort_msg = [m for m in pending_msg if isinstance(m, AbortMsg)] prompt_admitted_msg = [m for m in pending_msg if isinstance(m, PromptAdmittedMsg)] error_reply_msg = [m for m in pending_msg if isinstance(m, ErrorReplyMsg)] # Cache-rebuild control messages are pure passthrough (no tokenization): # CacheRebuildMsg (api -> scheduler) and CacheRebuildResultMsg (scheduler -> api). for m in pending_msg: if isinstance(m, CacheRebuildMsg): send_backend.put( CacheRebuildBackendMsg( request_id=m.request_id, moe_cache_size=m.moe_cache_size, num_pages=m.num_pages, num_mamba_slots=m.num_mamba_slots, num_swa_pages=m.num_swa_pages, mode=m.mode, ) ) elif isinstance(m, CacheRebuildResultMsg): send_frontend.put( CacheRebuildReply( request_id=m.request_id, status=m.status, moe_cache_size=m.moe_cache_size, num_pages=m.num_pages, mamba_slots=m.mamba_slots, num_swa_pages=m.num_swa_pages, error=m.error, ) ) elif isinstance(m, FlushCacheMsg): send_backend.put( FlushCacheBackendMsg(request_id=m.request_id, timeout=m.timeout) ) elif isinstance(m, FlushCacheResultMsg): send_frontend.put( FlushCacheReply(request_id=m.request_id, status=m.status, error=m.error) ) n_control = sum( isinstance( m, ( CacheRebuildMsg, CacheRebuildResultMsg, FlushCacheMsg, FlushCacheResultMsg, ErrorReplyMsg, PromptAdmittedMsg, ), ) for m in pending_msg ) assert ( len(detokenize_msg) + len(tokenize_msg) + len(abort_msg) + n_control == len(pending_msg) ) sampled_replies: List[UserReply] = [] if len(detokenize_msg) > 0: replies = detokenize_manager.detokenize(detokenize_msg) sampled_replies = [ UserReply( uid=msg.uid, incremental_output=reply, finished=msg.finished, finish_reason=msg.finish_reason, matched_stop=msg.matched_stop, completion_tokens_delta=len(msg.next_tokens), kv_used_pages=msg.kv_used_pages, kv_total_pages=msg.kv_total_pages, mamba_used_slots=msg.mamba_used_slots, mamba_total_slots=msg.mamba_total_slots, swa_used_tokens=msg.swa_used_tokens, swa_total_tokens=msg.swa_total_tokens, gpu_mem_bytes=msg.gpu_mem_bytes, timings=msg.timings, ) for msg, reply in zip(detokenize_msg, replies, strict=True) ] # An error reply and a client abort are both terminal for their uid, and neither # produces the finished DetokenizeMsg that would release the decode state. for msg in error_reply_msg: detokenize_manager.discard(msg.uid) for msg in abort_msg: detokenize_manager.discard(msg.uid) _send_generation_replies( send_frontend, [_prompt_admitted_reply(msg) for msg in prompt_admitted_msg], sampled_replies, [_error_reply(msg) for msg in error_reply_msg], ) if len(tokenize_msg) > 0: # Tokenize per-message so a single un-renderable request (e.g. a chat template # that rejects the message layout) becomes a terminal error reply for THAT uid # instead of an uncaught exception that kills the worker and bricks the server. backend, errors = _tokenize_requests(tokenize_manager, tokenize_msg, logger) if errors: send_frontend.put( errors[0] if len(errors) == 1 else BatchFrontendMsg(data=errors) ) if backend: send_backend.put(backend[0] if len(backend) == 1 else BatchBackendMsg(data=backend)) if len(abort_msg) > 0: batch_output = BatchBackendMsg( data=[AbortBackendMsg(uid=msg.uid) for msg in abort_msg] ) if len(batch_output.data) == 1: batch_output = batch_output.data[0] send_backend.put(batch_output) except KeyboardInterrupt: pass