Files
Xiaoze Fan 08d728d5e3 feat(mm): serve image input on the Qwen families (#454)
* feat(mm): serve image input on the Qwen VL families

- frontend: OpenAI image_url, Anthropic image blocks and Responses
  input_image (http(s), base64, gated file://) become MMItems in the
  tokenizer workers; image spans carry content pseudo ids so the radix
  cache keys them by content; mrope positions are precomputed per request
- registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next
  register a vision encoder serving image) and ModelSpec.mm_processor;
  EngineConfig.active_encoders decides what a process builds,
  served_modalities what it accepts; --text-model-only, --mm-disable,
  --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels
- engine and scheduler: per-request encoder-cache row ownership, chunk
  planning on the batch (jobs, gather rows, scatter rows), items encoded
  right before the LM forward and gathered into batch.mm_embeds
- models: the Qwen VL vision tower on the text tower's TP layers with
  DeepStack, block weights streamed from pinned host banks by default,
  merger outputs parked on the host between blocks; SupportsMultimodal
  (encode, place_encoder_weights) and a shared embed_input_ids
- server: image inputs rejected with a client-facing reason when no
  vision encoder is served; GET /v1/stats reports model.input_modalities

* feat(mm): replace the pixel cap with a token budget and processor kwargs

- --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge
- --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget
- MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve
- the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
2026-09-13 15:39:33 -07:00

123 lines
4.7 KiB
Python

"""Block weights streamed from pinned host banks through two device staging buffers: block i computes from buffer i % 2 while the copy stream fills the other with block i + 1."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterator, List, Sequence
import torch
from freetoken.layers import BaseOP
_ALIGN = 256
@dataclass(frozen=True)
class _Slot:
owner: Any
attr: str
offset: int
nbytes: int
dtype: torch.dtype
shape: torch.Size
def view(self, row: torch.Tensor) -> torch.Tensor:
return row[self.offset : self.offset + self.nbytes].view(self.dtype).view(self.shape)
def _slots(block: BaseOP) -> tuple[List[_Slot], int]:
"""Every tensor attribute of the block (recursively), laid out back to back with 256-byte alignment."""
slots: List[_Slot] = []
offset = 0
def walk(op: BaseOP) -> None:
nonlocal offset
for name, value in op.__dict__.items():
if name.startswith("_"):
continue
if isinstance(value, torch.Tensor):
nbytes = value.numel() * value.element_size()
slots.append(_Slot(op, name, offset, nbytes, value.dtype, value.shape))
offset += -(-nbytes // _ALIGN) * _ALIGN
elif isinstance(value, BaseOP):
walk(value)
walk(block)
return slots, offset
class BlockWeightStreamer:
"""Owns the host banks and the two staging buffers of a block stack; drives one forward with `blocks()`."""
def __init__(self, blocks: Sequence[BaseOP], device: torch.device):
self.device = device
self._layouts = [_slots(b) for b in blocks]
row_bytes = self._layouts[0][1]
assert all(nbytes == row_bytes for _, nbytes in self._layouts), "streamed blocks must share one layout"
self.bank = torch.empty((len(blocks), row_bytes), dtype=torch.uint8, pin_memory=True)
self.staging = torch.empty((2, row_bytes), dtype=torch.uint8, device=device)
for b, (slots, _) in enumerate(self._layouts):
for s in slots:
s.view(self.bank[b]).copy_(getattr(s.owner, s.attr))
torch.cuda.synchronize(device)
self._bind_all_to_host()
self.copy_stream = torch.cuda.Stream(device=device)
self.begin_event = torch.cuda.Event()
self.ready_events = [torch.cuda.Event() for _ in range(2)]
self.release_events = [torch.cuda.Event() for _ in range(2)]
self._holder: List[int | None] = [None, None]
self._has_release: List[bool] = [False, False]
@property
def device_bytes(self) -> int:
return self.staging.numel()
def _bind_all_to_host(self) -> None:
# between forwards the attributes point at the pinned bank, so state_dict() stays complete and correct
for b, (slots, _) in enumerate(self._layouts):
for s in slots:
setattr(s.owner, s.attr, s.view(self.bank[b]))
def unstream(self) -> None:
"""Put every block's tensors back on the device as plain resident copies."""
for b, (slots, _) in enumerate(self._layouts):
for s in slots:
setattr(s.owner, s.attr, s.view(self.bank[b]).to(self.device))
torch.cuda.synchronize(self.device)
def _prefetch(self, i: int) -> None:
if i >= len(self._layouts):
return
buf = i % 2
if self._holder[buf] == i:
return
with torch.cuda.stream(self.copy_stream):
if self._has_release[buf]:
self.copy_stream.wait_event(self.release_events[buf])
self.staging[buf].copy_(self.bank[i], non_blocking=True)
self.ready_events[buf].record(self.copy_stream)
self._holder[buf] = i
def blocks(self, ops: Sequence[BaseOP]) -> Iterator[tuple[int, BaseOP]]:
"""Yield (i, block) with block i's tensors bound to a filled staging buffer while block i + 1 streams in."""
compute = torch.cuda.current_stream(self.device)
# the copy stream must not overwrite staging that a still-running previous forward reads
self.begin_event.record(compute)
self.copy_stream.wait_event(self.begin_event)
for i, op in enumerate(ops):
buf = i % 2
self._prefetch(i)
self._prefetch(i + 1)
compute.wait_event(self.ready_events[buf])
slots, _ = self._layouts[i]
for s in slots:
setattr(s.owner, s.attr, s.view(self.staging[buf]))
yield i, op
self.release_events[buf].record(compute)
self._has_release[buf] = True
for s in slots:
setattr(s.owner, s.attr, s.view(self.bank[i]))
__all__ = ["BlockWeightStreamer"]