Files
Mukkesh Ganesh 23fc922bf8 feat(deepseek_v4): serve image input on the V4-Flash-Vision release (#502)
* feat(deepseek_v4): serve image input on the V4-Flash-Vision release

The block layout depends on where the image lands: lead pads align its first row token to
a 4-token compression stride, so the item hash carries the layout and two alignments never
share a radix key.

The block's tokens all carry one content pad id (the shared pad scheme); the sparse
attention reads its spans from batch.mm_spans, which the scheduler recovers from the flat
block ends its existing plan already reports.

* fix(mm): keep the shared mmprocessor signature and own the dsv4 offset in its apply

* fix(models): build dsv4 image spans in the model and start them at image_start

* fix(models): mount the dsv4 vision tower under visual and keep its weights unquantized

* fix(models): keep dsv4 image blocks whole across chunked prefill
2026-10-08 14:42:39 -07:00

124 lines
3.7 KiB
Python

"""DeepSeek-V4-Flash hyperparameters.
Field names mirror the authors' ``inference/config.json`` (consumed by the
reference ``ModelArgs``) so the port stays 1:1 with the reference. ``load_args``
reads that file from the checkpoint directory (it ships alongside the weights);
the few runtime knobs (batch / sequence length) are overlaid by the runner.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass, field, fields
from typing import Literal, Tuple
@dataclass
class DeepseekV4Args:
# ----- runtime -----
max_batch_size: int = 1
max_seq_len: int = 4096
dtype: Literal["bf16", "fp8"] = "fp8"
scale_fmt: Literal[None, "ue8m0"] = "ue8m0"
expert_dtype: Literal[None, "fp4"] = "fp4"
scale_dtype: Literal["fp32", "fp8"] = "fp8"
# ----- shape -----
vocab_size: int = 129280
dim: int = 4096
moe_inter_dim: int = 2048
n_layers: int = 43
n_hash_layers: int = 3
n_mtp_layers: int = 1
n_heads: int = 64
# ----- moe -----
n_routed_experts: int = 256
n_shared_experts: int = 1
n_activated_experts: int = 6
score_func: Literal["softmax", "sigmoid", "sqrtsoftplus"] = "sqrtsoftplus"
route_scale: float = 1.5
swiglu_limit: float = 10.0
# ----- mla -----
q_lora_rank: int = 1024
head_dim: int = 512
rope_head_dim: int = 64
norm_eps: float = 1e-6
o_groups: int = 8
o_lora_rank: int = 1024
window_size: int = 128
compress_ratios: Tuple[int, ...] = (0, 0, 4, 128, 4, 128, 4, 0)
# ----- rope / yarn -----
compress_rope_theta: float = 160000.0
original_seq_len: int = 65536
rope_theta: float = 10000.0
rope_factor: float = 16
beta_fast: int = 32
beta_slow: int = 1
# ----- lightning indexer -----
index_n_heads: int = 64
index_head_dim: int = 128
index_topk: int = 512
# ----- hyper-connections -----
hc_mult: int = 4
hc_sinkhorn_iters: int = 20
hc_eps: float = 1e-6
# ----- vision tower (DeepSeek-V4-Flash-Vision-Exp; 0 on the text-only releases) -----
vision_n_layers: int = 0
vision_dim: int = 1024
vision_n_heads: int = 16
vision_inter_dim: int = 2816
vision_patch_size: int = 14
vision_rope_theta: float = 10000.0
vision_downsample_ratio: int = 3
vision_max_n_token: int = 384
vision_min_pixels: int = 147456
vision_max_wh_ratio: int = 8
def __post_init__(self) -> None:
# JSON lists -> tuple so the dataclass stays hashable / immutable-ish.
if isinstance(self.compress_ratios, list):
self.compress_ratios = tuple(self.compress_ratios)
@property
def nope_head_dim(self) -> int:
return self.head_dim - self.rope_head_dim
def _config_path(model_path: str) -> str:
"""Locate the authors' ModelArgs JSON inside the checkpoint directory."""
candidates = [
os.path.join(model_path, "inference", "config.json"),
os.path.join(model_path, "model_args.json"),
]
for path in candidates:
if os.path.exists(path):
return path
raise FileNotFoundError(
f"No DeepSeek-V4 ModelArgs JSON found under {model_path} "
f"(looked for inference/config.json)"
)
def load_args(model_path: str, **overrides) -> DeepseekV4Args:
"""Build :class:`DeepseekV4Args` from the checkpoint's ``inference/config.json``.
``overrides`` (e.g. ``max_seq_len``, ``max_batch_size``) take precedence over the
file, letting the runner size the per-request caches.
"""
with open(_config_path(model_path)) as f:
raw = json.load(f)
valid = {f.name for f in fields(DeepseekV4Args)}
kwargs = {k: v for k, v in raw.items() if k in valid}
kwargs.update(overrides)
return DeepseekV4Args(**kwargs)
__all__ = ["DeepseekV4Args", "load_args"]