mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 10:02:33 +08:00
* feat(deepseek_v4): serve image input on the V4-Flash-Vision release The block layout depends on where the image lands: lead pads align its first row token to a 4-token compression stride, so the item hash carries the layout and two alignments never share a radix key. The block's tokens all carry one content pad id (the shared pad scheme); the sparse attention reads its spans from batch.mm_spans, which the scheduler recovers from the flat block ends its existing plan already reports. * fix(mm): keep the shared mmprocessor signature and own the dsv4 offset in its apply * fix(models): build dsv4 image spans in the model and start them at image_start * fix(models): mount the dsv4 vision tower under visual and keep its weights unquantized * fix(models): keep dsv4 image blocks whole across chunked prefill
519 lines
23 KiB
Python
519 lines
23 KiB
Python
"""MMProcessor: placeholder expansion on synthetic families, the Qwen VL processor and its positions, the registry (no checkpoint)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from freetoken.message import MMItem
|
|
from freetoken.mm import MM_PAD_SHIFT_VALUE, mm_pad_value
|
|
from freetoken.mm.config import MultimodalConfig
|
|
from freetoken.mm.processor import MMProcessor, PromptReplacement, image_positions
|
|
from freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessor
|
|
from freetoken.mm.processors.glm5_next import Glm5NextMMProcessor
|
|
from freetoken.mm.processors.muse_glimmer import MuseGlimmerMMProcessor
|
|
from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID, MiniMaxM3MMProcessor
|
|
from freetoken.mm.processors.qwen_vl import QwenVLMMProcessor
|
|
from freetoken.mm.processors.deepseek_v4 import DSV4MMProcessor
|
|
|
|
PLACEHOLDER = 7
|
|
BOI, SOFT, EOI = 90, 91, 92
|
|
ARCH = "Qwen3_5ForConditionalGeneration"
|
|
|
|
|
|
def _png():
|
|
from PIL import Image
|
|
|
|
buf = io.BytesIO()
|
|
Image.new("RGB", (8, 8)).save(buf, format="PNG")
|
|
return buf.getvalue()
|
|
|
|
|
|
class _Family(MMProcessor):
|
|
"""Fixed-size images wrapped in begin/end tokens; only the soft tokens take embeddings."""
|
|
|
|
placeholder = [PLACEHOLDER]
|
|
|
|
def __init__(self, n_soft):
|
|
super().__init__("/fake", MultimodalConfig())
|
|
self.n_soft = n_soft
|
|
|
|
def process(self, images):
|
|
return [
|
|
MMItem(modality="image", hash=100 + i, pad_value=MM_PAD_SHIFT_VALUE + 100 + i, offsets=[], feature=torch.zeros(1))
|
|
for i, _ in enumerate(images)
|
|
]
|
|
|
|
def prompt_replacement(self, item):
|
|
return PromptReplacement.select_token_id([BOI] + [SOFT] * self.n_soft + [EOI], SOFT)
|
|
|
|
def dummy_items(self, dtype, device):
|
|
raise NotImplementedError
|
|
|
|
|
|
class _Tiled(_Family):
|
|
"""Rows of soft tokens separated by a break token, like a tiled image."""
|
|
|
|
def prompt_replacement(self, item):
|
|
full = ([SOFT] * 2 + [BOI]) * 3
|
|
return PromptReplacement.select_token_id(full, SOFT)
|
|
|
|
|
|
def test_replacement_keeps_wrapper_tokens_and_pads_only_embedding_slots():
|
|
ids = torch.tensor([1, 2, PLACEHOLDER, 3, PLACEHOLDER, 4], dtype=torch.int32)
|
|
r = _Family(n_soft=3).apply(ids, [_png(), _png()])
|
|
a, b = r.mm_items
|
|
assert r.input_ids.tolist() == [1, 2, BOI, a.pad_value, a.pad_value, a.pad_value, EOI, 3, BOI, b.pad_value, b.pad_value, b.pad_value, EOI, 4]
|
|
assert a.offsets == [[3, 6]] and b.offsets == [[9, 12]]
|
|
assert a.num_tokens == 3 and r.mrope_positions is None and r.mrope_delta == 0
|
|
|
|
|
|
def test_replacement_with_several_embedding_runs_yields_several_spans():
|
|
ids = torch.tensor([PLACEHOLDER, 5], dtype=torch.int32)
|
|
r = _Tiled(n_soft=0).apply(ids, [_png()])
|
|
(item,) = r.mm_items
|
|
p = item.pad_value
|
|
assert r.input_ids.tolist() == [p, p, BOI, p, p, BOI, p, p, BOI, 5]
|
|
assert item.offsets == [[0, 2], [3, 5], [6, 8]] and item.num_tokens == 6
|
|
|
|
|
|
def test_placeholder_count_is_checked_before_any_image_is_decoded():
|
|
ids = torch.tensor([PLACEHOLDER, PLACEHOLDER], dtype=torch.int32)
|
|
with pytest.raises(ValueError, match="2 image placeholders but the request carries 1"):
|
|
_Family(n_soft=1).apply(ids, [b"not an image"])
|
|
|
|
|
|
def test_image_positions_freeze_t_spread_hw_and_advance_by_the_longer_side():
|
|
# [text x2][img 2x3 = 6][text x1][img 3x1 = 3][text x2] -> 14 tokens
|
|
pos, delta = image_positions(14, [(2, 6, 2, 3), (9, 3, 3, 1)])
|
|
assert pos[:, :2].tolist() == [[0, 1]] * 3
|
|
assert pos[0, 2:8].tolist() == [2] * 6
|
|
assert pos[1, 2:8].tolist() == [2, 2, 2, 3, 3, 3]
|
|
assert pos[2, 2:8].tolist() == [2, 3, 4, 2, 3, 4]
|
|
assert pos[:, 8].tolist() == [5, 5, 5] # 2 + max(2, 3)
|
|
assert pos[0, 9:12].tolist() == [6] * 3
|
|
assert pos[1, 9:12].tolist() == [6, 7, 8]
|
|
assert pos[2, 9:12].tolist() == [6, 6, 6]
|
|
assert pos[:, 12:].tolist() == [[9, 10]] * 3 # 6 + max(3, 1)
|
|
assert delta == 10 + 1 - 14
|
|
|
|
|
|
def _hf_config(vision=True, mrope=True, arch=ARCH):
|
|
rope = {"rope_theta": 1e7}
|
|
if mrope:
|
|
rope.update(mrope_section=[24, 20, 20], mrope_interleaved=True)
|
|
return SimpleNamespace(
|
|
architectures=[arch],
|
|
image_token_id=151655,
|
|
vision_config=(
|
|
SimpleNamespace(spatial_merge_size=2, patch_size=16, temporal_patch_size=2, in_channels=3)
|
|
if vision
|
|
else None
|
|
),
|
|
text_config=SimpleNamespace(rope_parameters=rope, vocab_size=248320),
|
|
)
|
|
|
|
|
|
def _grid_item(h, w, offsets):
|
|
return MMItem(
|
|
modality="image", hash=1, pad_value=1, offsets=offsets,
|
|
feature=torch.zeros(1), model_specific_data={"grid_thw": [1, h, w]},
|
|
)
|
|
|
|
|
|
def test_qwen_processor_reads_config():
|
|
proc = QwenVLMMProcessor(_hf_config(), "/nonexistent", MultimodalConfig())
|
|
assert proc.image_token_id == 151655 and proc.is_mrope and proc.merge == 2
|
|
assert proc.image_grid(_grid_item(8, 6, [[0, 12]])) == (4, 3)
|
|
repl = proc.prompt_replacement(_grid_item(8, 6, []))
|
|
assert proc.placeholder == [151655] and repl.full == [151655] * 12 and repl.embed_spans() == [[0, 12]]
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (4, 3 * 2 * 16 * 16) and dummy.num_tokens == 1
|
|
assert dummy.grid_thw == [1, 2, 2]
|
|
|
|
|
|
def test_qwen_token_budget_and_kwargs_reach_the_image_processor():
|
|
from freetoken.mm.config import MultimodalConfig
|
|
|
|
calls = []
|
|
|
|
class _FakeImageProcessor:
|
|
size = {"shortest_edge": 65536, "longest_edge": 16777216}
|
|
|
|
def __call__(self, images, **kwargs):
|
|
calls.append(kwargs)
|
|
return {"pixel_values": torch.zeros(4, 3 * 2 * 16 * 16), "image_grid_thw": torch.tensor([[1, 2, 2]])}
|
|
|
|
mm = MultimodalConfig(image_min_tokens=100, image_max_tokens=1000, processor_kwargs={"do_convert_rgb": False})
|
|
proc = QwenVLMMProcessor(_hf_config(), "/nonexistent", mm)
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
(item,) = proc.process([object()])
|
|
assert item.grid_thw == [1, 2, 2]
|
|
# one token is a 32x32 patch of the resized image; the budget travels as pixel areas in size, the extra kwarg rides along
|
|
assert calls == [{"return_tensors": "pt", "size": {"shortest_edge": 100 * 1024, "longest_edge": 1000 * 1024}, "do_convert_rgb": False}]
|
|
proc = QwenVLMMProcessor(_hf_config(), "/nonexistent", MultimodalConfig())
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
proc.process([object()])
|
|
assert calls[-1] == {"return_tensors": "pt"} # no budget, no size override
|
|
|
|
|
|
def test_qwen_processor_positions_follow_the_grid():
|
|
proc = QwenVLMMProcessor(_hf_config(), "/nonexistent", MultimodalConfig())
|
|
# 3 text tokens, one 8x6-patch image (4x3 = 12 llm tokens), 2 text tokens
|
|
pos, delta = proc.positions(17, [_grid_item(8, 6, [[3, 15]])])
|
|
assert pos.shape == (3, 17)
|
|
assert pos[:, :3].tolist() == [[0, 1, 2]] * 3
|
|
assert pos[0, 3:15].tolist() == [3] * 12
|
|
assert pos[1, 3:15].tolist() == [3 + i // 3 for i in range(12)]
|
|
assert pos[2, 3:15].tolist() == [3 + i % 3 for i in range(12)]
|
|
assert pos[:, 15:].tolist() == [[7, 8]] * 3 # resumes at 3 + max(4, 3)
|
|
assert delta == 8 + 1 - 17
|
|
|
|
|
|
def _gemma_config():
|
|
return SimpleNamespace(
|
|
architectures=["Gemma4ForConditionalGeneration"],
|
|
image_token_id=258880,
|
|
boi_token_id=255999,
|
|
eoi_token_id=258882,
|
|
vision_soft_tokens_per_image=280,
|
|
vision_config=SimpleNamespace(patch_size=16, pooling_kernel_size=3),
|
|
text_config=SimpleNamespace(vocab_size=262144),
|
|
)
|
|
|
|
|
|
class _FakeGemmaImageProcessor:
|
|
"""Two soft tokens (18 valid patches) padded out to a 2520-row batch, like the real processor."""
|
|
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def __call__(self, images, return_tensors, max_soft_tokens=None):
|
|
self.calls.append(max_soft_tokens)
|
|
rows = 2520
|
|
pixels = torch.zeros(1, rows, 768)
|
|
pixels[0, :18] = 0.75
|
|
positions = torch.full((1, rows, 2), -1, dtype=torch.int64)
|
|
positions[0, :18, 0] = torch.arange(18) % 6
|
|
positions[0, :18, 1] = torch.arange(18) // 6
|
|
return {"pixel_values": pixels, "image_position_ids": positions, "num_soft_tokens_per_image": torch.tensor([2])}
|
|
|
|
|
|
def _gemma_processor(mm=None):
|
|
proc = Gemma4MMProcessor(_gemma_config(), "/nonexistent", mm or MultimodalConfig())
|
|
fake = _FakeGemmaImageProcessor()
|
|
proc._image_processor = lambda: fake
|
|
return proc, fake
|
|
|
|
|
|
def test_gemma_items_strip_padding_and_wrap_the_soft_tokens_in_boi_eoi():
|
|
proc, _ = _gemma_processor()
|
|
(item,) = proc.process([object()])
|
|
assert item.feature.shape == (18, 768) and item.feature.dtype == torch.float32
|
|
assert item.feature[0, 0].item() == 0.75 # the [0, 1] pixels of the reference processor; the tower maps them to [-1, 1]
|
|
assert item.position_ids.shape == (18, 2) and item.position_ids[7].tolist() == [1, 1]
|
|
assert item.num_soft_tokens == 2 and item.pad_value >= MM_PAD_SHIFT_VALUE
|
|
repl = proc.prompt_replacement(item)
|
|
assert proc.placeholder == [258880]
|
|
assert repl.full == [255999, 258880, 258880, 258882] and repl.embed_spans() == [[1, 3]]
|
|
assert proc.positions(10, [item]) is None
|
|
|
|
|
|
def test_gemma_image_max_tokens_picks_an_accepted_soft_token_budget():
|
|
for budget, tier in ((910, 560), (70, 70), (None, None)): # the largest accepted budget within the limit
|
|
proc, fake = _gemma_processor(MultimodalConfig(image_max_tokens=budget))
|
|
proc.process([object()])
|
|
assert fake.calls == [tier]
|
|
with pytest.raises(ValueError, match="below the smallest"): # a maximum no budget honors is refused at start-up instead of rounded up
|
|
_gemma_processor(MultimodalConfig(image_max_tokens=60))
|
|
proc, _ = _gemma_processor(MultimodalConfig(image_min_tokens=500, processor_kwargs={"max_soft_tokens": 1120}))
|
|
# every image is scaled to its budget, so the minimum has no effect; an explicit kwarg wins over the budget
|
|
assert proc.get_mm_processor_kwargs(proc.mm) == {"return_tensors": "pt", "max_soft_tokens": 1120}
|
|
|
|
|
|
def test_gemma_dummy_item_is_one_pooled_soft_token():
|
|
proc, _ = _gemma_processor()
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (9, 768) and dummy.num_tokens == 1 and dummy.num_soft_tokens == 1
|
|
assert dummy.position_ids.tolist() == [[x, y] for y in range(3) for x in range(3)]
|
|
|
|
|
|
def _gemma_unified_config():
|
|
return SimpleNamespace(
|
|
architectures=["Gemma4UnifiedForConditionalGeneration"],
|
|
image_token_id=258880,
|
|
boi_token_id=255999,
|
|
eoi_token_id=258882,
|
|
vision_config=SimpleNamespace(patch_size=16, pooling_kernel_size=3, model_patch_size=48, num_soft_tokens=280),
|
|
text_config=SimpleNamespace(vocab_size=262144),
|
|
)
|
|
|
|
|
|
class _FakeGemmaUnifiedImageProcessor:
|
|
"""Six super-patches padded out to the budget's rows, like the real processor: one row per soft token."""
|
|
|
|
def __call__(self, images, return_tensors, max_soft_tokens=None):
|
|
rows = max_soft_tokens or 280
|
|
pixels = torch.zeros(1, rows, 6912)
|
|
pixels[0, :6] = 0.25
|
|
positions = torch.full((1, rows, 2), -1, dtype=torch.int64)
|
|
positions[0, :6, 0] = torch.arange(6) % 3
|
|
positions[0, :6, 1] = torch.arange(6) // 3
|
|
return {"pixel_values": pixels, "image_position_ids": positions, "num_soft_tokens_per_image": torch.tensor([6])}
|
|
|
|
|
|
def test_gemma_unified_items_are_one_row_per_soft_token():
|
|
proc = Gemma4UnifiedMMProcessor(_gemma_unified_config(), "/nonexistent", MultimodalConfig(image_max_tokens=100))
|
|
proc._image_processor = lambda: _FakeGemmaUnifiedImageProcessor()
|
|
(item,) = proc.process([object()])
|
|
assert item.feature.shape == (6, 6912) and item.num_soft_tokens == 6 and item.position_ids[4].tolist() == [1, 1]
|
|
assert proc.prompt_replacement(item).full == [255999] + [258880] * 6 + [258882]
|
|
assert proc.get_mm_processor_kwargs(proc.mm) == {"return_tensors": "pt", "max_soft_tokens": 70} # the same budgets as the tower release
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (1, 6912) and dummy.num_tokens == 1 and dummy.position_ids.tolist() == [[0, 0]]
|
|
|
|
|
|
def _glm_config():
|
|
return SimpleNamespace(
|
|
architectures=["Glm5NextForConditionalGeneration"],
|
|
image_token_id=154854,
|
|
vision_config=SimpleNamespace(spatial_merge_size=2, patch_size=14, temporal_patch_size=2, in_channels=3),
|
|
text_config=SimpleNamespace(vocab_size=154880),
|
|
)
|
|
|
|
|
|
def test_glm_token_budget_reaches_the_image_processor_and_rope_stays_one_dimensional():
|
|
calls = []
|
|
|
|
class _FakeImageProcessor:
|
|
def __call__(self, images, **kwargs):
|
|
calls.append(kwargs)
|
|
return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}
|
|
|
|
mm = MultimodalConfig(image_min_tokens=32, image_max_tokens=1024, processor_kwargs={"do_convert_rgb": False})
|
|
proc = Glm5NextMMProcessor(_glm_config(), "/nonexistent", mm)
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
(item,) = proc.process([object()])
|
|
# the budget is already in tokens; the extra kwarg rides along
|
|
assert calls == [{"return_tensors": "pt", "min_image_tokens": 32, "max_image_tokens": 1024, "do_convert_rgb": False}]
|
|
assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16
|
|
# the template renders the begin/end wrapper tokens itself; only <|image|> expands
|
|
assert proc.placeholder == [154854] and proc.prompt_replacement(item).full == [154854] * 4
|
|
assert proc.positions(10, [item]) is None
|
|
proc = Glm5NextMMProcessor(_glm_config(), "/nonexistent", MultimodalConfig())
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
proc.process([object()])
|
|
assert calls[-1] == {"return_tensors": "pt"}
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1
|
|
|
|
|
|
def _muse_config():
|
|
return SimpleNamespace(
|
|
architectures=["MuseGlimmerForConditionalGeneration"],
|
|
image_token_id=200092,
|
|
vision_config=SimpleNamespace(merge_size=2, patch_temporal=2, patch_size=14),
|
|
text_config=SimpleNamespace(vocab_size=202048),
|
|
)
|
|
|
|
|
|
def test_muse_wraps_the_pads_in_image_start_end_and_takes_only_a_token_maximum():
|
|
calls = []
|
|
|
|
class _FakeImageProcessor:
|
|
def __call__(self, images, **kwargs):
|
|
calls.append(kwargs)
|
|
return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}
|
|
|
|
mm = MultimodalConfig(image_min_tokens=32, image_max_tokens=1024, processor_kwargs={"do_convert_rgb": False})
|
|
proc = MuseGlimmerMMProcessor(_muse_config(), "/nonexistent", mm)
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
(item,) = proc.process([object()])
|
|
# the image processor has no minimum knob: only the maximum reaches it, the extra kwarg rides along
|
|
assert calls == [{"return_tensors": "pt", "max_image_tokens": 1024, "do_convert_rgb": False}]
|
|
assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16
|
|
# the template renders one <|patch|>; the replacement adds the start/end wrapper the checkpoint processor renders
|
|
repl = proc.prompt_replacement(item)
|
|
assert proc.placeholder == [200092] and repl.full == [200080, *[200092] * 4, 200081]
|
|
assert repl.embed_spans() == [[1, 5]] and proc.positions(10, [item]) is None
|
|
proc = MuseGlimmerMMProcessor(_muse_config(), "/nonexistent", MultimodalConfig())
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
proc.process([object()])
|
|
assert calls[-1] == {"return_tensors": "pt"}
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1
|
|
|
|
|
|
def _minimax_config():
|
|
# the checkpoint's own config shape: image_token_index, merge sizes under img_token_compression_config
|
|
return SimpleNamespace(
|
|
architectures=["MiniMaxM3SparseForConditionalGeneration"],
|
|
image_token_index=200025,
|
|
vision_config=SimpleNamespace(patch_size=14, num_channels=3, img_token_compression_config={"spatial_merge_size": 2, "temporal_patch_size": 2}),
|
|
text_config=SimpleNamespace(vocab_size=200064),
|
|
)
|
|
|
|
|
|
def test_minimax_token_budget_becomes_pixel_bounds_and_the_span_gets_start_end_tokens():
|
|
calls = []
|
|
|
|
class _FakeImageProcessor:
|
|
size = {"shortest_edge": 3136, "longest_edge": 451584}
|
|
|
|
def __call__(self, images, **kwargs):
|
|
calls.append(kwargs)
|
|
return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}
|
|
|
|
mm = MultimodalConfig(image_min_tokens=8, image_max_tokens=256, processor_kwargs={"do_convert_rgb": False})
|
|
proc = MiniMaxM3MMProcessor(_minimax_config(), "/nonexistent", mm)
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
(item,) = proc.process([object()])
|
|
# 784 pixels per token: patch 14 x merge 2, squared
|
|
assert calls == [{"return_tensors": "pt", "size": {"shortest_edge": 8 * 784, "longest_edge": 256 * 784}, "do_convert_rgb": False}]
|
|
assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16
|
|
repl = proc.prompt_replacement(item)
|
|
assert proc.placeholder == [200025] and repl.full == [IMAGE_START_ID] + [200025] * 4 + [IMAGE_END_ID] and repl.embed_spans() == [[1, 5]]
|
|
assert proc.positions(10, [item]) is None
|
|
proc = MiniMaxM3MMProcessor(_minimax_config(), "/nonexistent", MultimodalConfig())
|
|
proc._image_processor = lambda: _FakeImageProcessor()
|
|
proc.process([object()])
|
|
assert calls[-1] == {"return_tensors": "pt"}
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1
|
|
|
|
|
|
def test_registry_resolves_by_architecture(monkeypatch):
|
|
import freetoken.utils
|
|
from freetoken.mm.processor import get_mm_processor
|
|
|
|
configs = {
|
|
"/qwen": _hf_config(),
|
|
"/gemma": _gemma_config(),
|
|
"/gemma-unified": _gemma_unified_config(),
|
|
"/glm": _glm_config(),
|
|
"/muse": _muse_config(),
|
|
"/minimax": _minimax_config(),
|
|
"/text-only": _hf_config(vision=False),
|
|
"/unknown-vlm": _hf_config(arch="SomeOtherForConditionalGeneration"),
|
|
}
|
|
monkeypatch.setattr(freetoken.utils, "cached_load_hf_config", lambda p: configs[p])
|
|
assert isinstance(get_mm_processor("/qwen"), QwenVLMMProcessor)
|
|
assert get_mm_processor("/qwen", MultimodalConfig(disabled_encoders=frozenset({"vision"}))) is None # --mm-disable vision
|
|
assert isinstance(get_mm_processor("/gemma"), Gemma4MMProcessor)
|
|
assert isinstance(get_mm_processor("/gemma-unified"), Gemma4UnifiedMMProcessor)
|
|
assert isinstance(get_mm_processor("/glm"), Glm5NextMMProcessor)
|
|
assert isinstance(get_mm_processor("/muse"), MuseGlimmerMMProcessor)
|
|
assert isinstance(get_mm_processor("/minimax"), MiniMaxM3MMProcessor)
|
|
assert get_mm_processor("/text-only") is None
|
|
assert get_mm_processor("/unknown-vlm") is None
|
|
assert get_mm_processor("/missing") is None # a config that fails to load means no vision
|
|
|
|
|
|
def _dsv4_config(**over):
|
|
"""The vision dims sit at the top level of a deepseek_v4 checkpoint config."""
|
|
base = dict(
|
|
vision_n_layers=32,
|
|
vision_dim=1024,
|
|
vision_n_heads=16,
|
|
vision_inter_dim=2816,
|
|
vision_patch_size=14,
|
|
vision_rope_theta=10000.0,
|
|
vision_downsample_ratio=3,
|
|
vision_max_n_token=384,
|
|
vision_min_pixels=147456,
|
|
vision_max_wh_ratio=8,
|
|
hidden_size=4096,
|
|
)
|
|
return SimpleNamespace(**{**base, **over})
|
|
|
|
|
|
def _dsv4_image(width, height):
|
|
from PIL import Image
|
|
|
|
return Image.new("RGB", (width, height))
|
|
|
|
|
|
def test_dsv4_block_length_and_hash_follow_the_insertion_offset():
|
|
from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID
|
|
from freetoken.models.deepseek_v4.vision import build_image_block
|
|
|
|
proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
|
|
assert proc.placeholder == [IMAGE_TOKEN_ID]
|
|
(item,) = proc.process([_dsv4_image(640, 480)])
|
|
assert (item.n_vit_h, item.n_vit_w, item.n_llm_h, item.n_llm_w) == (35, 46, 12, 16)
|
|
|
|
repl = proc.prompt_replacement(item, 0)
|
|
at_zero, hash_zero = len(repl.full), item.hash
|
|
assert at_zero == len(build_image_block(12, 16, 0)[0]) == 209
|
|
assert repl.full == [IMAGE_TOKEN_ID] * at_zero
|
|
# the block's sentinels are embeddings too, so the whole span takes image rows
|
|
assert repl.embed_spans() == [[0, at_zero]]
|
|
assert item.pad_value == mm_pad_value(item.hash) and item.pad_value >= MM_PAD_SHIFT_VALUE
|
|
|
|
# the lead pads align the block to a compression stride, so a later offset is a shorter
|
|
# block AND a different content hash: one alignment's rows must never serve another's
|
|
repl = proc.prompt_replacement(item, 2)
|
|
assert len(repl.full) == at_zero - 2 and item.hash != hash_zero
|
|
assert item.pad_value == mm_pad_value(item.hash) >= MM_PAD_SHIFT_VALUE
|
|
|
|
|
|
def test_dsv4_dummy_item_is_the_smallest_encodable_block():
|
|
from freetoken.models.deepseek_v4.vision import build_image_block
|
|
|
|
proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
|
|
(dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
|
|
dummy.validate()
|
|
assert dummy.feature.shape == (36, 3, 14, 14) # a 6x6 patch grid, one 2x2 merge grid
|
|
assert dummy.num_tokens == len(build_image_block(2, 2, 0)[0]) == 13
|
|
|
|
|
|
def test_a_text_only_checkpoint_config_has_no_dsv4_processor():
|
|
proc = DSV4MMProcessor
|
|
with pytest.raises(ValueError):
|
|
proc(_dsv4_config(vision_n_layers=0), "/nonexistent", MultimodalConfig())
|
|
|
|
|
|
def test_dsv4_apply_passes_each_block_the_offset_it_lands_at():
|
|
"""Only the DSV4 block depends on its position, so its own apply computes the offset."""
|
|
from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID
|
|
from freetoken.models.deepseek_v4.vision import build_image_block
|
|
|
|
proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
|
|
buf = io.BytesIO()
|
|
_dsv4_image(640, 480).save(buf, format="PNG")
|
|
raw = buf.getvalue()
|
|
ids = torch.tensor([1, IMAGE_TOKEN_ID, 2, IMAGE_TOKEN_ID, 3], dtype=torch.int32)
|
|
out = proc.apply(ids, [raw, raw])
|
|
first, second = out.mm_items
|
|
assert (first.n_llm_h, first.n_llm_w) == (12, 16)
|
|
# the first placeholder sits at 1, the second after the text between plus the first block
|
|
assert first.start == first.offsets[0][0] == 1
|
|
assert second.start == second.offsets[0][0]
|
|
assert first.num_tokens == len(build_image_block(12, 16, first.start)[0])
|
|
assert second.num_tokens == len(build_image_block(12, 16, second.start)[0])
|
|
# apply fills the embedding slots with each block's content pad id, not the placeholder
|
|
for item in (first, second):
|
|
lo, hi = item.offsets[0]
|
|
assert set(out.input_ids[lo:hi].tolist()) == {item.pad_value}
|
|
assert IMAGE_TOKEN_ID not in out.input_ids.tolist()
|
|
|
|
|
|
def test_dsv4_image_max_tokens_lowers_the_checkpoints_block_cap():
|
|
from freetoken.models.deepseek_v4.vision import build_image_block
|
|
|
|
big = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
|
|
capped = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig(image_max_tokens=64))
|
|
(a,) = big.process([_dsv4_image(1600, 1200)])
|
|
(b,) = capped.process([_dsv4_image(1600, 1200)])
|
|
assert b.n_llm_h * b.n_llm_w < a.n_llm_h * a.n_llm_w
|
|
assert len(build_image_block(b.n_llm_h, b.n_llm_w, 0)[0]) <= 64
|