mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(mm): serve image input on the Qwen VL families - frontend: OpenAI image_url, Anthropic image blocks and Responses input_image (http(s), base64, gated file://) become MMItems in the tokenizer workers; image spans carry content pseudo ids so the radix cache keys them by content; mrope positions are precomputed per request - registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next register a vision encoder serving image) and ModelSpec.mm_processor; EngineConfig.active_encoders decides what a process builds, served_modalities what it accepts; --text-model-only, --mm-disable, --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels - engine and scheduler: per-request encoder-cache row ownership, chunk planning on the batch (jobs, gather rows, scatter rows), items encoded right before the LM forward and gathered into batch.mm_embeds - models: the Qwen VL vision tower on the text tower's TP layers with DeepStack, block weights streamed from pinned host banks by default, merger outputs parked on the host between blocks; SupportsMultimodal (encode, place_encoder_weights) and a shared embed_input_ids - server: image inputs rejected with a client-facing reason when no vision encoder is served; GET /v1/stats reports model.input_modalities * feat(mm): replace the pixel cap with a token budget and processor kwargs - --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge - --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget - MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve - the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
123 lines
4.7 KiB
Python
123 lines
4.7 KiB
Python
"""Block weights streamed from pinned host banks through two device staging buffers: block i computes from buffer i % 2 while the copy stream fills the other with block i + 1."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any, Iterator, List, Sequence
|
|
|
|
import torch
|
|
|
|
from freetoken.layers import BaseOP
|
|
|
|
_ALIGN = 256
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _Slot:
|
|
owner: Any
|
|
attr: str
|
|
offset: int
|
|
nbytes: int
|
|
dtype: torch.dtype
|
|
shape: torch.Size
|
|
|
|
def view(self, row: torch.Tensor) -> torch.Tensor:
|
|
return row[self.offset : self.offset + self.nbytes].view(self.dtype).view(self.shape)
|
|
|
|
|
|
def _slots(block: BaseOP) -> tuple[List[_Slot], int]:
|
|
"""Every tensor attribute of the block (recursively), laid out back to back with 256-byte alignment."""
|
|
slots: List[_Slot] = []
|
|
offset = 0
|
|
|
|
def walk(op: BaseOP) -> None:
|
|
nonlocal offset
|
|
for name, value in op.__dict__.items():
|
|
if name.startswith("_"):
|
|
continue
|
|
if isinstance(value, torch.Tensor):
|
|
nbytes = value.numel() * value.element_size()
|
|
slots.append(_Slot(op, name, offset, nbytes, value.dtype, value.shape))
|
|
offset += -(-nbytes // _ALIGN) * _ALIGN
|
|
elif isinstance(value, BaseOP):
|
|
walk(value)
|
|
|
|
walk(block)
|
|
return slots, offset
|
|
|
|
|
|
class BlockWeightStreamer:
|
|
"""Owns the host banks and the two staging buffers of a block stack; drives one forward with `blocks()`."""
|
|
|
|
def __init__(self, blocks: Sequence[BaseOP], device: torch.device):
|
|
self.device = device
|
|
self._layouts = [_slots(b) for b in blocks]
|
|
row_bytes = self._layouts[0][1]
|
|
assert all(nbytes == row_bytes for _, nbytes in self._layouts), "streamed blocks must share one layout"
|
|
self.bank = torch.empty((len(blocks), row_bytes), dtype=torch.uint8, pin_memory=True)
|
|
self.staging = torch.empty((2, row_bytes), dtype=torch.uint8, device=device)
|
|
for b, (slots, _) in enumerate(self._layouts):
|
|
for s in slots:
|
|
s.view(self.bank[b]).copy_(getattr(s.owner, s.attr))
|
|
torch.cuda.synchronize(device)
|
|
self._bind_all_to_host()
|
|
self.copy_stream = torch.cuda.Stream(device=device)
|
|
self.begin_event = torch.cuda.Event()
|
|
self.ready_events = [torch.cuda.Event() for _ in range(2)]
|
|
self.release_events = [torch.cuda.Event() for _ in range(2)]
|
|
self._holder: List[int | None] = [None, None]
|
|
self._has_release: List[bool] = [False, False]
|
|
|
|
@property
|
|
def device_bytes(self) -> int:
|
|
return self.staging.numel()
|
|
|
|
def _bind_all_to_host(self) -> None:
|
|
# between forwards the attributes point at the pinned bank, so state_dict() stays complete and correct
|
|
for b, (slots, _) in enumerate(self._layouts):
|
|
for s in slots:
|
|
setattr(s.owner, s.attr, s.view(self.bank[b]))
|
|
|
|
def unstream(self) -> None:
|
|
"""Put every block's tensors back on the device as plain resident copies."""
|
|
for b, (slots, _) in enumerate(self._layouts):
|
|
for s in slots:
|
|
setattr(s.owner, s.attr, s.view(self.bank[b]).to(self.device))
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
def _prefetch(self, i: int) -> None:
|
|
if i >= len(self._layouts):
|
|
return
|
|
buf = i % 2
|
|
if self._holder[buf] == i:
|
|
return
|
|
with torch.cuda.stream(self.copy_stream):
|
|
if self._has_release[buf]:
|
|
self.copy_stream.wait_event(self.release_events[buf])
|
|
self.staging[buf].copy_(self.bank[i], non_blocking=True)
|
|
self.ready_events[buf].record(self.copy_stream)
|
|
self._holder[buf] = i
|
|
|
|
def blocks(self, ops: Sequence[BaseOP]) -> Iterator[tuple[int, BaseOP]]:
|
|
"""Yield (i, block) with block i's tensors bound to a filled staging buffer while block i + 1 streams in."""
|
|
compute = torch.cuda.current_stream(self.device)
|
|
# the copy stream must not overwrite staging that a still-running previous forward reads
|
|
self.begin_event.record(compute)
|
|
self.copy_stream.wait_event(self.begin_event)
|
|
for i, op in enumerate(ops):
|
|
buf = i % 2
|
|
self._prefetch(i)
|
|
self._prefetch(i + 1)
|
|
compute.wait_event(self.ready_events[buf])
|
|
slots, _ = self._layouts[i]
|
|
for s in slots:
|
|
setattr(s.owner, s.attr, s.view(self.staging[buf]))
|
|
yield i, op
|
|
self.release_events[buf].record(compute)
|
|
self._has_release[buf] = True
|
|
for s in slots:
|
|
setattr(s.owner, s.attr, s.view(self.bank[i]))
|
|
|
|
|
|
__all__ = ["BlockWeightStreamer"]
|