mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
134 lines
5.1 KiB
Python
134 lines
5.1 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Final, List
|
|
|
|
import torch
|
|
from freetoken.message import BaseBackendMsg, BaseTokenizerMsg, BatchTokenizerMsg
|
|
from freetoken.utils import ZmqPubQueue, ZmqPullQueue, ZmqPushQueue, ZmqSubQueue, init_logger
|
|
|
|
if TYPE_CHECKING:
|
|
from .config import SchedulerConfig
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class SchedulerIOMixin:
|
|
"""
|
|
Mixin class for Scheduler I/O operations.
|
|
|
|
This class handles the communication between the scheduler and the tokenizer.
|
|
|
|
Public Utilities:
|
|
receive_msg: Function to receive messages from the tokenizer.
|
|
send_result: Function to send results back to the tokenizer.
|
|
sync_all_ranks: Function to synchronize all ranks on CPU side.
|
|
"""
|
|
|
|
def __init__(self, config: SchedulerConfig, tp_cpu_group: torch.distributed.ProcessGroup):
|
|
tp_info = config.tp_info
|
|
self.tp_cpu_group: Final = tp_cpu_group
|
|
if config.offline_mode:
|
|
self.receive_msg = self.offline_receive_msg
|
|
self.send_result = self.offline_send_result
|
|
return # early exit
|
|
|
|
if tp_info.is_primary():
|
|
self._recv_from_tokenizer: Final = ZmqPullQueue(
|
|
config.zmq_backend_addr,
|
|
create=True,
|
|
decoder=BaseBackendMsg.decoder,
|
|
)
|
|
self._send_into_tokenizer: Final = ZmqPushQueue(
|
|
config.zmq_detokenizer_addr,
|
|
create=config.backend_create_detokenizer_link,
|
|
encoder=BaseTokenizerMsg.encoder,
|
|
)
|
|
|
|
recv = self._recv_msg_single_rank
|
|
send = self._reply_tokenizer_rank0
|
|
if tp_info.size > 1:
|
|
if tp_info.is_primary():
|
|
recv = self._recv_msg_multi_rank0
|
|
self._send_into_ranks: Final = ZmqPubQueue(
|
|
config.zmq_scheduler_broadcast_addr, create=True, encoder=BaseBackendMsg.encoder
|
|
)
|
|
else:
|
|
recv = self._recv_msg_multi_rank1
|
|
send = self._reply_tokenizer_rank1
|
|
self._recv_from_rank0: Final = ZmqSubQueue(
|
|
config.zmq_scheduler_broadcast_addr,
|
|
create=False,
|
|
decoder=BaseBackendMsg.decoder,
|
|
)
|
|
|
|
self.receive_msg = recv
|
|
self.send_result = send
|
|
|
|
def run_when_idle(self):
|
|
raise NotImplementedError("should be implemented")
|
|
|
|
def offline_receive_msg(self, blocking: bool = False) -> List[BaseBackendMsg]:
|
|
raise NotImplementedError("should be implemented")
|
|
|
|
def offline_send_result(self, reply: List[BaseTokenizerMsg]) -> None:
|
|
raise NotImplementedError("should be implemented")
|
|
|
|
def sync_all_ranks(self) -> None:
|
|
self.tp_cpu_group.barrier().wait()
|
|
|
|
def _recv_msg_single_rank(self, blocking: bool = False) -> List[BaseBackendMsg]:
|
|
pending_msgs: List[BaseBackendMsg] = []
|
|
if blocking:
|
|
self.run_when_idle()
|
|
pending_msgs.append(self._recv_from_tokenizer.get())
|
|
while not self._recv_from_tokenizer.empty():
|
|
pending_msgs.append(self._recv_from_tokenizer.get())
|
|
return pending_msgs
|
|
|
|
def _recv_msg_multi_rank0(self, blocking: bool = False) -> List[BaseBackendMsg]:
|
|
pending_msgs: List[BaseBackendMsg] = []
|
|
if blocking:
|
|
self.run_when_idle()
|
|
raw = self._recv_from_tokenizer.get_raw()
|
|
self._send_into_ranks.put_raw(raw)
|
|
pending_msgs.append(self._recv_from_tokenizer.decode(raw))
|
|
|
|
pending_raw_msgs: List[bytes] = []
|
|
while not self._recv_from_tokenizer.empty():
|
|
pending_raw_msgs.append(self._recv_from_tokenizer.get_raw())
|
|
|
|
# broadcast the number of raw messages to all ranks
|
|
src_tensor = torch.tensor(len(pending_raw_msgs))
|
|
self.tp_cpu_group.broadcast(src_tensor, root=0).wait()
|
|
|
|
for raw in pending_raw_msgs:
|
|
self._send_into_ranks.put_raw(raw)
|
|
pending_msgs.append(self._recv_from_tokenizer.decode(raw))
|
|
return pending_msgs
|
|
|
|
def _recv_msg_multi_rank1(self, blocking: bool = False) -> List[BaseBackendMsg]:
|
|
pending_msgs: List[BaseBackendMsg] = []
|
|
if blocking:
|
|
self.run_when_idle()
|
|
pending_msgs.append(self._recv_from_rank0.get())
|
|
|
|
# ensure all ranks have the same number of raw messages
|
|
dst_tensor = torch.tensor(-1)
|
|
self.tp_cpu_group.broadcast(dst_tensor, root=0).wait()
|
|
dst_length = int(dst_tensor.item())
|
|
|
|
for _ in range(dst_length):
|
|
pending_msgs.append(self._recv_from_rank0.get())
|
|
return pending_msgs
|
|
|
|
def _reply_tokenizer_rank0(self, reply: List[BaseTokenizerMsg]) -> None:
|
|
num_reply = len(reply)
|
|
logger.debug_rank0(f"Replying to tokenizer: {num_reply} messages")
|
|
if num_reply == 1:
|
|
self._send_into_tokenizer.put(reply[0])
|
|
elif num_reply > 1:
|
|
self._send_into_tokenizer.put(BatchTokenizerMsg(data=reply)) # type: ignore
|
|
|
|
def _reply_tokenizer_rank1(self, reply: List[BaseTokenizerMsg]) -> None:
|
|
_ = reply # do nothing for non-primary ranks
|