mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(mm): serve image input on the Qwen VL families - frontend: OpenAI image_url, Anthropic image blocks and Responses input_image (http(s), base64, gated file://) become MMItems in the tokenizer workers; image spans carry content pseudo ids so the radix cache keys them by content; mrope positions are precomputed per request - registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next register a vision encoder serving image) and ModelSpec.mm_processor; EngineConfig.active_encoders decides what a process builds, served_modalities what it accepts; --text-model-only, --mm-disable, --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels - engine and scheduler: per-request encoder-cache row ownership, chunk planning on the batch (jobs, gather rows, scatter rows), items encoded right before the LM forward and gathered into batch.mm_embeds - models: the Qwen VL vision tower on the text tower's TP layers with DeepStack, block weights streamed from pinned host banks by default, merger outputs parked on the host between blocks; SupportsMultimodal (encode, place_encoder_weights) and a shared embed_input_ids - server: image inputs rejected with a client-facing reason when no vision encoder is served; GET /v1/stats reports model.input_modalities * feat(mm): replace the pixel cap with a token budget and processor kwargs - --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge - --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget - MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve - the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
93 lines
3.3 KiB
Python
93 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any, Dict, Type
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
|
|
_TYPE_KEY = "__type__"
|
|
# Message payloads carry free-form client dicts (tool JSON Schemas, chat_template_kwargs) that
|
|
# may legitimately use our tag key as a field name. Wrapping such a dict keeps the decoder from
|
|
# reading it as a serialized class -- without this, a request could crash the tokenizer worker.
|
|
_RAW_DICT_KEY = "__raw_dict__"
|
|
|
|
def _serialize_any(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
encoded = {k: _serialize_any(v) for k, v in value.items()}
|
|
if _TYPE_KEY in encoded or _RAW_DICT_KEY in encoded:
|
|
return {_RAW_DICT_KEY: encoded}
|
|
return encoded
|
|
elif isinstance(value, (list, tuple)):
|
|
return type(value)(_serialize_any(v) for v in value)
|
|
elif isinstance(value, (int, float, str, type(None), bool, bytes)):
|
|
return value
|
|
else:
|
|
return serialize_type(value)
|
|
|
|
|
|
def serialize_type(self) -> Dict:
|
|
# find all member variables
|
|
serialized = {}
|
|
|
|
if isinstance(self, torch.Tensor):
|
|
assert not self.is_cuda, "wire tensors must live on CPU"
|
|
t = self.contiguous()
|
|
serialized["__type__"] = "Tensor"
|
|
serialized["dtype"] = str(t.dtype)
|
|
# 1-D tensors omit the shape so the payload matches the legacy wire format.
|
|
if t.dim() != 1:
|
|
serialized["shape"] = list(t.shape)
|
|
if t.dtype == torch.bfloat16:
|
|
t = t.view(torch.uint16) # numpy has no bf16; ship the raw bytes
|
|
serialized["buffer"] = t.numpy().tobytes()
|
|
return serialized
|
|
|
|
# normal type
|
|
serialized["__type__"] = self.__class__.__name__
|
|
for k, v in self.__dict__.items():
|
|
serialized[k] = _serialize_any(v)
|
|
return serialized
|
|
|
|
|
|
def _deserialize_any(cls_map: Dict[str, Type], data: Any) -> Any:
|
|
if isinstance(data, dict):
|
|
if len(data) == 1 and _RAW_DICT_KEY in data:
|
|
inner = data[_RAW_DICT_KEY]
|
|
return {k: _deserialize_any(cls_map, v) for k, v in inner.items()}
|
|
if _TYPE_KEY in data:
|
|
return deserialize_type(cls_map, data)
|
|
else:
|
|
return {k: _deserialize_any(cls_map, v) for k, v in data.items()}
|
|
elif isinstance(data, (list, tuple)):
|
|
return type(data)(_deserialize_any(cls_map, d) for d in data)
|
|
elif isinstance(data, (int, float, str, type(None), bool, bytes)):
|
|
return data
|
|
else:
|
|
raise ValueError(f"Cannot deserialize type {type(data)}")
|
|
|
|
|
|
def deserialize_type(cls_map: Dict[str, Type], data: Dict) -> Any:
|
|
type_name = data["__type__"]
|
|
if type_name == "Tensor":
|
|
buffer = data["buffer"]
|
|
dtype_str = data["dtype"].replace("torch.", "")
|
|
assert isinstance(buffer, bytes)
|
|
is_bf16 = dtype_str == "bfloat16"
|
|
np_tensor = np.frombuffer(buffer, dtype=getattr(np, "uint16" if is_bf16 else dtype_str))
|
|
tensor = torch.from_numpy(np_tensor.copy())
|
|
if is_bf16:
|
|
tensor = tensor.view(torch.bfloat16)
|
|
shape = data.get("shape")
|
|
return tensor if shape is None else tensor.view(shape)
|
|
|
|
cls = cls_map.get(type_name)
|
|
if cls is None:
|
|
raise ValueError(f"Unknown serialized message type {type_name!r}")
|
|
kwargs = {}
|
|
for k, v in data.items():
|
|
if k == _TYPE_KEY:
|
|
continue
|
|
kwargs[k] = _deserialize_any(cls_map, v)
|
|
return cls(**kwargs)
|