Files
FreeToken/tests/engine/test_mm_encoder.py
Xiaoze Fan 08d728d5e3 feat(mm): serve image input on the Qwen families (#454)
* feat(mm): serve image input on the Qwen VL families

- frontend: OpenAI image_url, Anthropic image blocks and Responses
  input_image (http(s), base64, gated file://) become MMItems in the
  tokenizer workers; image spans carry content pseudo ids so the radix
  cache keys them by content; mrope positions are precomputed per request
- registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next
  register a vision encoder serving image) and ModelSpec.mm_processor;
  EngineConfig.active_encoders decides what a process builds,
  served_modalities what it accepts; --text-model-only, --mm-disable,
  --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels
- engine and scheduler: per-request encoder-cache row ownership, chunk
  planning on the batch (jobs, gather rows, scatter rows), items encoded
  right before the LM forward and gathered into batch.mm_embeds
- models: the Qwen VL vision tower on the text tower's TP layers with
  DeepStack, block weights streamed from pinned host banks by default,
  merger outputs parked on the host between blocks; SupportsMultimodal
  (encode, place_encoder_weights) and a shared embed_input_ids
- server: image inputs rejected with a client-facing reason when no
  vision encoder is served; GET /v1/stats reports model.input_modalities

* feat(mm): replace the pixel cap with a token budget and processor kwargs

- --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge
- --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget
- MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve
- the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
2026-09-13 15:39:33 -07:00

97 lines
3.3 KiB
Python

"""Engine._run_mm_encoder against a fake model: chunked gathers, a shared image, precomputed embeddings, orphan jobs."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from freetoken.engine.engine import Engine
from freetoken.message.backend import MMItem
from freetoken.mm.encoder_cache import EncoderCache
H = 8
class _FakeVision:
def __init__(self):
self.calls = 0
def encode(self, item: MMItem) -> torch.Tensor:
self.calls += 1
return torch.full((item.num_tokens, H), float(item.hash))
def _engine(cache: EncoderCache, model=None) -> SimpleNamespace:
return SimpleNamespace(
encoder_cache=cache, device=torch.device("cpu"), dtype=torch.float32,
model=model or _FakeVision(),
)
def _item(h: int, n_tokens: int, precomputed: torch.Tensor | None = None) -> MMItem:
feature = None if precomputed is not None else torch.zeros(1)
return MMItem(
modality="image", hash=h, pad_value=0, offsets=[[0, n_tokens]],
feature=feature, precomputed_embeddings=precomputed,
)
def _batch(jobs, plan) -> SimpleNamespace:
return SimpleNamespace(mm_encoder_jobs=jobs, mm_gather_plan=plan, mm_embeds=None, mm_rows=None)
def test_entry_lives_until_its_consumer_gathers_the_last_row():
cache = EncoderCache(storage="cpu")
eng = _engine(cache)
item = _item(h=3, n_tokens=4)
cache.register(3, 7, 4) # admission claims the whole image
# chunk 1 consumes rows [0, 2) of a 4-token image
Engine._run_mm_encoder(eng, _batch([item], [(7, 3, 0, 2, 4, 0)]))
assert cache._entries[3].remaining == {7: 2}
assert item.feature is None
# chunk 2 finishes the image; the entry dies with its last claim
Engine._run_mm_encoder(eng, _batch([], [(7, 3, 2, 4, 4, 0)]))
assert not cache.has(3)
assert eng.model.calls == 1
def test_shared_image_is_encoded_once_and_sliced_per_request():
cache = EncoderCache(storage="cpu")
eng = _engine(cache)
a, b = _item(h=5, n_tokens=4), _item(h=5, n_tokens=4)
cache.register(5, 1, 4)
cache.register(5, 2, 4)
batch = _batch([a, b], [(1, 5, 0, 4, 4, 0), (2, 5, 0, 2, 4, 4)])
Engine._run_mm_encoder(eng, batch)
assert eng.model.calls == 1
assert batch.mm_embeds.shape == (6, H)
assert torch.equal(batch.mm_embeds, torch.full((6, H), 5.0))
# request 1 consumed its image; request 2 still has rows to gather in its next chunk
assert cache._entries[5].remaining == {2: 2}
def test_precomputed_embeddings_bypass_the_encoder():
cache = EncoderCache(storage="cpu")
class _NoVision:
def encode(self, item):
raise AssertionError("encoder must not run for precomputed embeddings")
eng = _engine(cache, model=_NoVision())
emb = torch.arange(2 * H, dtype=torch.float32).view(2, H)
item = _item(h=4, n_tokens=2, precomputed=emb)
cache.register(4, 1, 2)
batch = _batch([item], [(1, 4, 0, 2, 2, 0)])
Engine._run_mm_encoder(eng, batch)
assert torch.equal(batch.mm_embeds, emb)
assert item.precomputed_embeddings is None
assert not cache.has(4)
def test_job_without_gather_row_fails_loudly():
eng = _engine(EncoderCache(storage="cpu"))
with pytest.raises(AssertionError, match="encoder jobs without gather rows"):
Engine._run_mm_encoder(eng, _batch([_item(h=1, n_tokens=2)], []))