mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(mm): serve image input on the Qwen VL families - frontend: OpenAI image_url, Anthropic image blocks and Responses input_image (http(s), base64, gated file://) become MMItems in the tokenizer workers; image spans carry content pseudo ids so the radix cache keys them by content; mrope positions are precomputed per request - registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next register a vision encoder serving image) and ModelSpec.mm_processor; EngineConfig.active_encoders decides what a process builds, served_modalities what it accepts; --text-model-only, --mm-disable, --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels - engine and scheduler: per-request encoder-cache row ownership, chunk planning on the batch (jobs, gather rows, scatter rows), items encoded right before the LM forward and gathered into batch.mm_embeds - models: the Qwen VL vision tower on the text tower's TP layers with DeepStack, block weights streamed from pinned host banks by default, merger outputs parked on the host between blocks; SupportsMultimodal (encode, place_encoder_weights) and a shared embed_input_ids - server: image inputs rejected with a client-facing reason when no vision encoder is served; GET /v1/stats reports model.input_modalities * feat(mm): replace the pixel cap with a token budget and processor kwargs - --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge - --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget - MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve - the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
34 lines
1.7 KiB
Python
34 lines
1.7 KiB
Python
"""Runtime knobs of the multimodal path. The architecture side (vision_config, mrope) lives in ModelConfig."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Literal
|
|
|
|
# encoder tower kinds a family can register; --mm-disable <kind> leaves that tower unbuilt and refuses its inputs
|
|
ENCODER_KINDS = ("vision", "audio")
|
|
# checkpoint config sections that describe encoder towers; the parser never sees the section of a tower this process does not build
|
|
ENCODER_SECTIONS = ("vision_config", "audio_config")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MultimodalConfig:
|
|
# encoder kinds (ENCODER_KINDS) whose tower this process does not build; --text-model-only names them all
|
|
disabled_encoders: frozenset[str] = frozenset()
|
|
# Where encoded items wait between prefill chunks. "cpu": pinned host memory, "cuda": the device.
|
|
embed_cache_device: Literal["cpu", "cuda"] = "cpu"
|
|
# Encoder tower block weights. "host": pinned host banks streamed two blocks at a time behind the compute, "gpu": resident.
|
|
encoder_weights: Literal["gpu", "host"] = "host"
|
|
# per-image token budget; the family's MMProcessor converts it to its image processor's own limits, None keeps the checkpoint defaults
|
|
image_min_tokens: int | None = None
|
|
image_max_tokens: int | None = None
|
|
# extra keyword arguments the family's MMProcessor passes to the image processor call, after the token budget
|
|
processor_kwargs: dict[str, Any] = field(default_factory=dict)
|
|
|
|
@property
|
|
def text_model_only(self) -> bool:
|
|
return set(ENCODER_KINDS) <= self.disabled_encoders
|
|
|
|
|
|
__all__ = ["ENCODER_KINDS", "ENCODER_SECTIONS", "MultimodalConfig"]
|