mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(deepseek_v4): serve image input on the V4-Flash-Vision release The block layout depends on where the image lands: lead pads align its first row token to a 4-token compression stride, so the item hash carries the layout and two alignments never share a radix key. The block's tokens all carry one content pad id (the shared pad scheme); the sparse attention reads its spans from batch.mm_spans, which the scheduler recovers from the flat block ends its existing plan already reports. * fix(mm): keep the shared mmprocessor signature and own the dsv4 offset in its apply * fix(models): build dsv4 image spans in the model and start them at image_start * fix(models): mount the dsv4 vision tower under visual and keep its weights unquantized * fix(models): keep dsv4 image blocks whole across chunked prefill
51 lines
2.1 KiB
Python
51 lines
2.1 KiB
Python
"""DeepSeek-V4 image preprocessing: one image -> ViT patches plus its grid metadata.
|
|
|
|
Port of the reference ``inference/image_processor.py:load_image``. The PIL work stays on the
|
|
CPU and must be bit-identical across runs: an image's radix key rides on these patches.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image, ImageOps
|
|
|
|
from .config import DSV4VisionConfig
|
|
from .vision import safe_resize
|
|
|
|
|
|
def load_image(image: Image.Image, vc: DSV4VisionConfig, max_n_token: int | None = None) -> tuple[torch.Tensor, int, int, int, int]:
|
|
"""One image -> ``(patches bf16 [n_vit_h*n_vit_w, 3, p, p], n_vit_h, n_vit_w, n_llm_h, n_llm_w)``.
|
|
|
|
``max_n_token`` lowers the checkpoint's per-image block cap (``--image-max-tokens``).
|
|
"""
|
|
image = image.convert("RGB")
|
|
p = vc.vision_patch_size
|
|
width, height = image.size
|
|
if vc.vision_max_wh_ratio is not None and width > height * vc.vision_max_wh_ratio:
|
|
width = height * vc.vision_max_wh_ratio
|
|
if 0 < width * height < vc.vision_min_pixels:
|
|
ratio = (vc.vision_min_pixels / (width * height)) ** 0.5
|
|
width = int(width * ratio)
|
|
height = int(height * ratio)
|
|
best_width = math.ceil(width / p) * p
|
|
best_height = math.ceil(height / p) * p
|
|
n_llm_h, n_llm_w, best_height, best_width = safe_resize(
|
|
height, width, best_height, best_width, p, vc.vision_downsample_ratio,
|
|
vc.vision_max_n_token if max_n_token is None else min(vc.vision_max_n_token, max_n_token),
|
|
)
|
|
n_vit_h, n_vit_w = best_height // p, best_width // p
|
|
if vc.vision_max_wh_ratio is not None and image.width >= vc.vision_max_wh_ratio * image.height:
|
|
image = image.resize((best_width, best_height))
|
|
else:
|
|
image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127))
|
|
x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255
|
|
x = ((x - 0.5) / 0.5).to(torch.bfloat16)
|
|
patches = x.reshape(3, n_vit_h, p, n_vit_w, p).permute(1, 3, 0, 2, 4).reshape(n_vit_h * n_vit_w, 3, p, p)
|
|
return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w
|
|
|
|
|
|
__all__ = ["load_image"]
|