mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(gguf): load the vision tower from a llama.cpp mmproj Qwen3.5 / Qwen3.6 / Qwen3.8 (qwen3vl_merger) and Gemma-4 26B-A4B / 31B (gemma4v) / 12B (gemma4uv) take image input from a separate mmproj file: --mmproj, else one in the model's own folder. The tower runs in bf16 (quantized mmproj files are dequantized) and its config comes from the GGUF metadata plus per-arch defaults. * docs(gguf): describe mmproj image input without a model list * test(gguf): drop the mmproj tests * fix(gguf): escape the model folder when globbing for mmproj files * fix(gguf): skip auto-discovered mmproj files the text model cannot use * fix(gguf): serve text from a GGUF that carries vision tensors itself
217 lines
8.6 KiB
Python
217 lines
8.6 KiB
Python
"""Per-family multimodal processors and the architecture registry; MMProcessor.apply is the image half of tokenization."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
import hashlib
|
|
import importlib
|
|
import io
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from freetoken.message import MMItem
|
|
from freetoken.mm.config import MultimodalConfig
|
|
from freetoken.mm import check_mm_pad_shift
|
|
|
|
|
|
@dataclass
|
|
class PromptReplacement:
|
|
"""The token sequence that replaces one placeholder; is_embed marks the positions that take image embeddings (None = all of them)."""
|
|
|
|
full: list[int]
|
|
is_embed: list[bool] | None = None
|
|
|
|
@classmethod
|
|
def select_token_id(cls, full: list[int], embed_token_id: int) -> PromptReplacement:
|
|
return cls(full, [t == embed_token_id for t in full])
|
|
|
|
def embed_spans(self) -> list[list[int]]:
|
|
"""Half-open [lo, hi) runs of embedding positions, relative to the start of full."""
|
|
mask = self.is_embed if self.is_embed is not None else [True] * len(self.full)
|
|
spans: list[list[int]] = []
|
|
lo: int | None = None
|
|
for i, flag in enumerate([*mask, False]):
|
|
if flag and lo is None:
|
|
lo = i
|
|
elif not flag and lo is not None:
|
|
spans.append([lo, i])
|
|
lo = None
|
|
return spans
|
|
|
|
|
|
@dataclass
|
|
class MMResult:
|
|
input_ids: torch.Tensor
|
|
mm_items: list[MMItem]
|
|
mrope_positions: torch.Tensor | None
|
|
mrope_delta: int
|
|
|
|
|
|
def _find_all(ids: list[int], target: list[int]) -> list[int]:
|
|
"""Start index of every non-overlapping occurrence of target."""
|
|
n = len(target)
|
|
found: list[int] = []
|
|
i = 0
|
|
while i <= len(ids) - n:
|
|
if ids[i : i + n] == target:
|
|
found.append(i)
|
|
i += n
|
|
else:
|
|
i += 1
|
|
return found
|
|
|
|
|
|
class MMProcessor(ABC):
|
|
"""How a checkpoint's images become items, tokens and positions; apply is the family-agnostic driver, subclasses fill the hooks.
|
|
|
|
Built from the checkpoint config alone and import-light: tokenizer workers call apply without the model layer, the engine calls dummy_items.
|
|
Image preprocessing must stay on the CPU with one fixed implementation: radix keys and the embedding cache are content hashes of pixel_values, and resize results differ across backends/devices.
|
|
"""
|
|
|
|
# the token sequence the chat template renders once per image
|
|
placeholder: list[int]
|
|
|
|
def __init__(self, model_path: str, mm: MultimodalConfig) -> None:
|
|
self.model_path = model_path
|
|
self.mm = mm
|
|
self._image_processor_instance: Any = None
|
|
self._image_processor_lock = threading.Lock()
|
|
|
|
def _image_processor(self) -> Any:
|
|
"""The checkpoint's image processor, loaded on first use (tokenizer workers share the instance across threads)."""
|
|
with self._image_processor_lock:
|
|
if self._image_processor_instance is None:
|
|
from freetoken.utils import load_image_processor
|
|
|
|
self._image_processor_instance = load_image_processor(self.model_path)
|
|
return self._image_processor_instance
|
|
|
|
def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:
|
|
"""Keyword arguments for the checkpoint processor call: the family's reading of the mm knobs, then mm.processor_kwargs on top; families with a token budget override this."""
|
|
return dict(mm.processor_kwargs)
|
|
|
|
@abstractmethod
|
|
def process(self, images: list[Any]) -> list[MMItem]:
|
|
"""One MMItem per image with feature and hash; offsets are assigned by apply."""
|
|
|
|
@abstractmethod
|
|
def prompt_replacement(self, item: MMItem) -> PromptReplacement:
|
|
"""The token sequence that stands in for this item's placeholder in the prompt."""
|
|
|
|
def positions(self, length: int, items: list[MMItem]) -> tuple[torch.Tensor, int] | None:
|
|
"""[3, length] t/h/w rope positions and the decode delta; None for 1-D rope families."""
|
|
return None
|
|
|
|
@abstractmethod
|
|
def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
|
|
"""The smallest encodable item of each modality the family serves, for warming the encoders."""
|
|
|
|
def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult:
|
|
"""Swap each image placeholder for its replacement sequence and precompute rope positions."""
|
|
from PIL import Image
|
|
|
|
ids = input_ids.tolist()
|
|
target = self.placeholder
|
|
slots = _find_all(ids, target)
|
|
if len(slots) != len(images):
|
|
raise ValueError(
|
|
f"prompt renders {len(slots)} image placeholders but the request "
|
|
f"carries {len(images)} images"
|
|
)
|
|
|
|
pils = [Image.open(io.BytesIO(raw)).convert("RGB") for raw in images]
|
|
items = self.process(pils)
|
|
out: list[int] = []
|
|
cursor = 0
|
|
for slot, item in zip(slots, items):
|
|
repl = self.prompt_replacement(item)
|
|
out.extend(ids[cursor:slot])
|
|
base = len(out)
|
|
full = list(repl.full)
|
|
spans = repl.embed_spans()
|
|
# embedding slots carry the content pad id so radix keys and the model's scatter mask see the image
|
|
for lo, hi in spans:
|
|
full[lo:hi] = [item.pad_value] * (hi - lo)
|
|
out.extend(full)
|
|
item.offsets = [[base + lo, base + hi] for lo, hi in spans]
|
|
item.validate()
|
|
cursor = slot + len(target)
|
|
out.extend(ids[cursor:])
|
|
new_ids = torch.tensor(out, dtype=input_ids.dtype)
|
|
|
|
positions, delta = self.positions(len(out), items) or (None, 0)
|
|
return MMResult(new_ids, items, positions, delta)
|
|
|
|
|
|
def image_positions(
|
|
length: int, blocks: list[tuple[int, int, int, int]]
|
|
) -> tuple[torch.Tensor, int]:
|
|
"""t/h/w positions: text advances all three rows, an image freezes t and spreads h/w over its grid, then the running position jumps by max(h, w).
|
|
|
|
blocks are (offset, n_tokens, grid_h, grid_w) in prompt order; delta shifts decode positions (sequence index + delta)."""
|
|
pos = torch.empty((3, length), dtype=torch.int32)
|
|
st = 0
|
|
cursor = 0
|
|
for offset, n_tokens, h_m, w_m in blocks:
|
|
run = offset - cursor
|
|
if run:
|
|
text = torch.arange(st, st + run, dtype=torch.int32)
|
|
pos[:, cursor:offset] = text
|
|
st += run
|
|
cursor = offset
|
|
idx = torch.arange(n_tokens, dtype=torch.int32)
|
|
pos[0, cursor : cursor + n_tokens] = st
|
|
pos[1, cursor : cursor + n_tokens] = st + idx // w_m
|
|
pos[2, cursor : cursor + n_tokens] = st + idx % w_m
|
|
st += max(h_m, w_m)
|
|
cursor += n_tokens
|
|
run = length - cursor
|
|
if run:
|
|
pos[:, cursor:] = torch.arange(st, st + run, dtype=torch.int32)
|
|
st += run
|
|
delta = int(pos.max().item()) + 1 - length
|
|
return pos, delta
|
|
|
|
|
|
def content_hash(feature: torch.Tensor, extra: bytes = b"") -> int:
|
|
"""62-bit hash of the wire bytes plus family extras (e.g. the grid, so transposed images differ)."""
|
|
buf = feature.contiguous()
|
|
if buf.dtype == torch.bfloat16:
|
|
buf = buf.view(torch.uint16)
|
|
hasher = hashlib.sha256(memoryview(buf.numpy()))
|
|
hasher.update(extra)
|
|
return int.from_bytes(hasher.digest()[:8], "little") % (1 << 62)
|
|
|
|
|
|
def get_mm_processor(model_path: str, mm: MultimodalConfig | None = None) -> MMProcessor | None:
|
|
"""A processor for the checkpoint under the mm knobs, or None when no encoder is left to serve: the family registers none, the checkpoint ships none, or mm.disabled_encoders names them all; callers keep the instance, it holds the loaded image processor."""
|
|
from freetoken.models.register import get_model_spec
|
|
from freetoken.utils import cached_load_hf_config
|
|
|
|
try:
|
|
config = cached_load_hf_config(model_path)
|
|
spec = get_model_spec((config.architectures or [None])[0])
|
|
except Exception: # noqa: BLE001 -- missing/foreign config or unknown architecture: no multimodal input
|
|
return None
|
|
mm = mm or MultimodalConfig()
|
|
served = [e for e in spec.encoders if getattr(config, e.config_key, None) is not None and e.kind not in mm.disabled_encoders]
|
|
if spec.mm_processor is None or not served:
|
|
return None
|
|
# families with a flat checkpoint config (deepseek_v4) carry vocab_size at the top level
|
|
check_mm_pad_shift(getattr(config, "text_config", config).vocab_size)
|
|
module, _, cls = spec.mm_processor.partition(":")
|
|
return getattr(importlib.import_module(module), cls)(config, model_path, mm)
|
|
|
|
|
|
__all__ = [
|
|
"MMProcessor",
|
|
"MMResult",
|
|
"PromptReplacement",
|
|
"content_hash",
|
|
"get_mm_processor",
|
|
"image_positions",
|
|
]
|