mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 10:02:33 +08:00
* feat(mm): serve image input on the Qwen VL families - frontend: OpenAI image_url, Anthropic image blocks and Responses input_image (http(s), base64, gated file://) become MMItems in the tokenizer workers; image spans carry content pseudo ids so the radix cache keys them by content; mrope positions are precomputed per request - registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next register a vision encoder serving image) and ModelSpec.mm_processor; EngineConfig.active_encoders decides what a process builds, served_modalities what it accepts; --text-model-only, --mm-disable, --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels - engine and scheduler: per-request encoder-cache row ownership, chunk planning on the batch (jobs, gather rows, scatter rows), items encoded right before the LM forward and gathered into batch.mm_embeds - models: the Qwen VL vision tower on the text tower's TP layers with DeepStack, block weights streamed from pinned host banks by default, merger outputs parked on the host between blocks; SupportsMultimodal (encode, place_encoder_weights) and a shared embed_input_ids - server: image inputs rejected with a client-facing reason when no vision encoder is served; GET /v1/stats reports model.input_modalities * feat(mm): replace the pixel cap with a token budget and processor kwargs - --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge - --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget - MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve - the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
"""EncoderCache ownership: rows claimed at admission, freed when the last claim is gathered."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
|
|
from freetoken.mm.encoder_cache import EncoderCache
|
|
|
|
CPU = torch.device("cpu")
|
|
|
|
|
|
def test_entry_lives_until_every_claim_is_gathered():
|
|
cache = EncoderCache(storage="cpu")
|
|
emb = torch.arange(12, dtype=torch.float32).reshape(6, 2)
|
|
cache.register(7, 1, 6)
|
|
cache.register(7, 2, 6)
|
|
assert not cache.has(7) # claimed, not encoded yet
|
|
cache.put(7, emb)
|
|
assert cache.has(7)
|
|
assert torch.equal(cache.get_slice(7, 2, 4, CPU), emb[2:4])
|
|
cache.consume(7, 1, 4)
|
|
cache.consume(7, 1, 2)
|
|
assert cache.has(7) # request 2 still has rows to gather
|
|
cache.consume(7, 2, 6)
|
|
assert not cache.has(7) and cache.stats() == (0, 0)
|
|
|
|
|
|
def test_repeated_image_in_one_request_adds_up():
|
|
cache = EncoderCache(storage="cpu")
|
|
cache.register(7, 1, 4)
|
|
cache.register(7, 1, 4)
|
|
cache.put(7, torch.ones(4, 2))
|
|
cache.consume(7, 1, 4) # the first occurrence is done
|
|
assert cache.has(7)
|
|
cache.consume(7, 1, 4)
|
|
assert not cache.has(7)
|
|
|
|
|
|
def test_release_drops_a_claim_even_before_the_tensor_arrives():
|
|
cache = EncoderCache(storage="cpu")
|
|
cache.register(5, 1, 3)
|
|
cache.register(5, 2, 3)
|
|
cache.release(1, [5, 99]) # unknown hash: no-op
|
|
assert 5 in cache._entries
|
|
cache.release(2, [5])
|
|
assert cache._entries == {} and cache.stats() == (0, 0)
|