Files
FreeToken/python/freetoken/mm/encoder_cache.py
Xiaoze Fan 08d728d5e3 feat(mm): serve image input on the Qwen families (#454)
* feat(mm): serve image input on the Qwen VL families

- frontend: OpenAI image_url, Anthropic image blocks and Responses
  input_image (http(s), base64, gated file://) become MMItems in the
  tokenizer workers; image spans carry content pseudo ids so the radix
  cache keys them by content; mrope positions are precomputed per request
- registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next
  register a vision encoder serving image) and ModelSpec.mm_processor;
  EngineConfig.active_encoders decides what a process builds,
  served_modalities what it accepts; --text-model-only, --mm-disable,
  --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels
- engine and scheduler: per-request encoder-cache row ownership, chunk
  planning on the batch (jobs, gather rows, scatter rows), items encoded
  right before the LM forward and gathered into batch.mm_embeds
- models: the Qwen VL vision tower on the text tower's TP layers with
  DeepStack, block weights streamed from pinned host banks by default,
  merger outputs parked on the host between blocks; SupportsMultimodal
  (encode, place_encoder_weights) and a shared embed_input_ids
- server: image inputs rejected with a client-facing reason when no
  vision encoder is served; GET /v1/stats reports model.input_modalities

* feat(mm): replace the pixel cap with a token budget and processor kwargs

- --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge
- --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget
- MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve
- the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
2026-09-13 15:39:33 -07:00

88 lines
3.2 KiB
Python

"""Encoder embedding cache keyed by content hash; an entry lives while any request still has rows of it to gather."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict
import torch
from freetoken.utils import init_logger
logger = init_logger(__name__)
@dataclass
class _Entry:
tensor: torch.Tensor | None = None
# uid -> embedding rows the request still has to gather, summed over its occurrences of the image
remaining: Dict[int, int] = field(default_factory=dict)
class EncoderCache:
def __init__(self, storage: str = "cpu") -> None:
assert storage in ("cpu", "cuda")
self._storage = storage
self._entries: Dict[int, _Entry] = {}
def register(self, item_hash: int, uid: int, rows: int) -> None:
"""Claim rows of the image for uid before any of its chunks run; a repeated image adds up."""
if rows <= 0:
return
entry = self._entries.setdefault(item_hash, _Entry())
entry.remaining[uid] = entry.remaining.get(uid, 0) + rows
def has(self, item_hash: int) -> bool:
entry = self._entries.get(item_hash)
return entry is not None and entry.tensor is not None
def put(self, item_hash: int, embedding: torch.Tensor) -> None:
entry = self._entries.get(item_hash)
assert entry is not None, f"image {item_hash} encoded before any request registered it"
if entry.tensor is not None:
return
if self._storage == "cpu":
stored = torch.empty(
embedding.shape, dtype=embedding.dtype, device="cpu",
pin_memory=torch.cuda.is_available(),
)
stored.copy_(embedding, non_blocking=True)
else:
stored = embedding
entry.tensor = stored
def get_slice(self, item_hash: int, lo: int, hi: int, device: torch.device) -> torch.Tensor:
view = self._entries[item_hash].tensor[lo:hi]
if view.device == device:
return view
return view.to(device, non_blocking=True)
def consume(self, item_hash: int, uid: int, rows: int) -> None:
"""Account rows uid gathered; the last row of the last holder frees the entry."""
entry = self._entries[item_hash]
left = entry.remaining[uid] - rows
assert left >= 0, f"request {uid} gathered more rows of image {item_hash} than it registered"
if left:
entry.remaining[uid] = left
return
del entry.remaining[uid]
if not entry.remaining:
del self._entries[item_hash]
def release(self, uid: int, hashes: list[int]) -> None:
"""Drop uid's claims whatever it gathered (abort); entries nobody else holds are freed."""
for h in hashes:
entry = self._entries.get(h)
if entry is None:
continue
entry.remaining.pop(uid, None)
if not entry.remaining:
del self._entries[h]
def stats(self) -> tuple[int, int]:
stored = [e.tensor for e in self._entries.values() if e.tensor is not None]
return len(stored), sum(t.numel() * t.element_size() for t in stored)
__all__ = ["EncoderCache"]