mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
302 lines
12 KiB
Python
302 lines
12 KiB
Python
"""Reasoning-effort dialect handling.
|
|
|
|
Each checkpoint's template/encoder accepts only its own effort vocabulary and
|
|
hard-fails on the rest, while clients speak whatever dialect their provider
|
|
taught them. Named levels project onto the numeric scale vLLM and SGLang share,
|
|
and out-of-vocabulary values quantize to the nearest supported gear instead of
|
|
failing the request. The vocabulary is probed from the checkpoint's own
|
|
template, never from a static table: a parser-family registry cannot be keyed
|
|
correctly (Qwen3 and Qwen3.8 resolve to the same parser but only the latter
|
|
grades effort).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any, Callable
|
|
|
|
#: Values must match vLLM's and SGLang's inkling table so the ecosystems agree
|
|
#: on what "medium" means relative to "xhigh".
|
|
EFFORT_SCALE: dict[str, float] = {
|
|
"none": 0.0,
|
|
"minimal": 0.1,
|
|
"low": 0.2,
|
|
"medium": 0.7,
|
|
"high": 0.9,
|
|
"xhigh": 0.99,
|
|
"max": 0.99,
|
|
}
|
|
|
|
KNOWN_REASONING_EFFORTS = tuple(EFFORT_SCALE)
|
|
|
|
|
|
#: OpenAI's effort triple -- the common-denominator vocabulary every dialect
|
|
#: understands.
|
|
OPENAI_EFFORT_TRIPLE = ("low", "medium", "high")
|
|
|
|
#: The graded ladder served to a template that grades effort WITHOUT validating
|
|
#: it (muse-glimmer's "Reasoning strength"). Such a template interpolates any
|
|
#: string, so its probed ``supported`` set is the whole scale -- but the
|
|
#: checkpoint's trained levels are the ladder names, and never-advertised
|
|
#: dialects (minimal / none / max) must quantize onto them instead of reaching
|
|
#: the model verbatim.
|
|
GRADED_EFFORT_LADDER = ("low", "medium", "high", "xhigh")
|
|
|
|
|
|
def effective_efforts(profile: EffortProfile) -> frozenset[str]:
|
|
"""The vocabulary a checkpoint is actually served (and advertised) with.
|
|
|
|
A NARROWED ``supported`` set is authoritative -- narrowing only ever comes
|
|
from observed rejections. A grader that accepted the whole scale validated
|
|
nothing, so pass-through would interpolate off-vocabulary values
|
|
("minimal", "max") into the prompt verbatim: it is capped to its dialect's
|
|
ladder instead -- the strength dialect's documented levels top out at xhigh
|
|
(muse-glimmer's model card), a plain effort grader keeps the OpenAI triple
|
|
(the only vocabulary such a template is known to understand)."""
|
|
if not profile.consumes_effort:
|
|
return frozenset()
|
|
if profile.validates or profile.supported != frozenset(KNOWN_REASONING_EFFORTS):
|
|
return profile.supported
|
|
ladder = GRADED_EFFORT_LADDER if profile.strength_dialect else OPENAI_EFFORT_TRIPLE
|
|
vocab = {name for name in ladder if name in profile.supported}
|
|
if profile.default:
|
|
vocab.add(profile.default)
|
|
return frozenset(vocab)
|
|
|
|
#: Protocol-level thinking toggles, broadcast in every spelling the ecosystem's
|
|
#: templates read (``enable_thinking`` bool: qwen/glm/gemma/dsv4;
|
|
#: ``thinking_mode`` string: minimax-m3). Jinja ignores undeclared variables,
|
|
#: so a template simply picks the knob it knows -- no per-family routing needed.
|
|
THINKING_OFF_KWARGS = {"enable_thinking": False, "thinking_mode": "disabled"}
|
|
THINKING_ON_KWARGS = {"enable_thinking": True, "thinking_mode": "enabled"}
|
|
THINKING_ADAPTIVE_KWARGS = {"thinking_mode": "adaptive"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EffortProfile:
|
|
"""What one checkpoint's template/encoder accepts, learned by probing it.
|
|
|
|
``default`` is the supported name whose rendering is byte-identical to
|
|
passing no effort at all. ``consumes_effort`` False means no probe round
|
|
ever changed its output or raised -- the template ignores the knob, so
|
|
requests should not carry it. ``validates`` True means the probe observed a
|
|
rejection: only then is ``supported`` a real vocabulary rather than "this
|
|
template interpolates anything". ``strength_dialect`` True means the
|
|
template reads the ``reasoning_strength`` spelling (muse-glimmer's family,
|
|
whose documented ladder tops out at xhigh) rather than only
|
|
``reasoning_effort``. ``integer_range`` records a validated 1-100 budget
|
|
accepted independently of the named effort vocabulary.
|
|
"""
|
|
|
|
supported: frozenset[str]
|
|
default: str | None
|
|
consumes_effort: bool
|
|
validates: bool = False
|
|
strength_dialect: bool = False
|
|
integer_range: tuple[int, int] | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ThinkingProfile:
|
|
"""A checkpoint's thinking controls, learned by probing its template.
|
|
|
|
``toggleable``: the off/on broadcasts render differently. ``has_adaptive``:
|
|
thinking_mode "adaptive" is a third distinct state (minimax-m3).
|
|
``default_state``: which state the bare render matches ("on" when not
|
|
toggleable or unmatched).
|
|
"""
|
|
|
|
efforts: EffortProfile
|
|
toggleable: bool
|
|
has_adaptive: bool
|
|
default_state: str # "on" | "off" | "adaptive"
|
|
|
|
|
|
#: A gear farther than this on the scale misrepresents the request; drop the
|
|
#: value and let the template default apply. Keeps OpenAI's "medium" from
|
|
#: escalating to the DSV4 encoder's absolute-maximum "high" gear (0.2 away),
|
|
#: matching vLLM's DSV4 mapping, while "high" still reaches Qwen's "xhigh"
|
|
#: (0.09 away).
|
|
_MAX_QUANTIZE_DISTANCE = 0.15
|
|
|
|
|
|
def quantize_effort(value: Any, profile: EffortProfile) -> str | int | None:
|
|
"""Map a client's effort onto ``profile``; ``None`` means "send nothing".
|
|
|
|
In-vocabulary values pass through untouched. Other named levels land on the
|
|
nearest supported gear within ``_MAX_QUANTIZE_DISTANCE`` -- except "max",
|
|
which is reachable only by its own name (vLLM's DSV4 rule: an extreme
|
|
opt-in gear must never be entered by rounding). Everything else drops to
|
|
the template default. With "max" excluded the remaining scale values are
|
|
unique, so quantization is deterministic across processes.
|
|
"""
|
|
if profile.integer_range is not None and isinstance(value, (int, float)):
|
|
low, high = profile.integer_range
|
|
if type(value) is not int or not low <= value <= high:
|
|
raise ValueError(f"reasoning_effort must be an integer in [{low}, {high}]")
|
|
return value
|
|
supported = effective_efforts(profile)
|
|
if not supported:
|
|
return None
|
|
if isinstance(value, str) and value in supported:
|
|
return value
|
|
position = EFFORT_SCALE.get(value) if isinstance(value, str) else None
|
|
if position is None:
|
|
return None
|
|
ranked = sorted(
|
|
(name for name in supported if name != "max"),
|
|
key=lambda name: (abs(EFFORT_SCALE[name] - position), -EFFORT_SCALE[name]),
|
|
)
|
|
if ranked and abs(EFFORT_SCALE[ranked[0]] - position) <= _MAX_QUANTIZE_DISTANCE:
|
|
return ranked[0]
|
|
return None
|
|
|
|
|
|
#: One tool flips tool-conditional paths (the DSV4 encoder grades effort only
|
|
#: in thinking mode, which tools force).
|
|
_PROBE_TOOLS = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "noop",
|
|
"description": "No-op probe tool.",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
]
|
|
|
|
#: (extra chat_template_kwargs, tools) per probe round: templates read effort
|
|
#: unconditionally, under tool-forced thinking, or only under an explicit
|
|
#: thinking opt-in -- one round per shape.
|
|
_PROBE_ROUNDS: tuple[tuple[dict[str, Any], list[dict[str, Any]] | None], ...] = (
|
|
({}, None),
|
|
({}, _PROBE_TOOLS),
|
|
({"enable_thinking": True}, None),
|
|
)
|
|
|
|
|
|
def probe_effort_profile(
|
|
render: Callable[[dict[str, Any], list[dict[str, Any]] | None], Any],
|
|
) -> EffortProfile:
|
|
"""Learn a checkpoint's effort vocabulary by rendering probes through it.
|
|
|
|
``render(chat_template_kwargs, tools)`` returns a comparable rendering and
|
|
raises on rejection. A round whose no-effort baseline raises is skipped:
|
|
the template rejected the probe conversation shape, not the effort.
|
|
"""
|
|
rejected: set[str] = set()
|
|
diverged: set[str] = set()
|
|
matches_baseline: dict[str, bool] = {name: True for name in KNOWN_REASONING_EFFORTS}
|
|
ran_rounds = 0
|
|
integer_range = None
|
|
|
|
for base_kwargs, tools in _PROBE_ROUNDS:
|
|
try:
|
|
baseline = render(dict(base_kwargs), tools)
|
|
except Exception: # noqa: BLE001 -- template rejects the probe shape, not the effort
|
|
continue
|
|
ran_rounds += 1
|
|
for name in KNOWN_REASONING_EFFORTS:
|
|
try:
|
|
rendering = render({**base_kwargs, "reasoning_effort": name}, tools)
|
|
except Exception: # noqa: BLE001 -- any raise means "not accepted"
|
|
rejected.add(name)
|
|
matches_baseline[name] = False
|
|
continue
|
|
if _renderings_differ(rendering, baseline):
|
|
diverged.add(name)
|
|
matches_baseline[name] = False
|
|
# Only a validated budget range is a numeric capability; arbitrary Jinja
|
|
# interpolation must not advertise support for numeric reasoning effort.
|
|
if _accepts_integer_budget(render, base_kwargs, tools):
|
|
integer_range = (1, 100)
|
|
|
|
if ran_rounds == 0:
|
|
# Nothing learnable: sending no effort is the only safe rendering.
|
|
return EffortProfile(supported=frozenset(), default=None, consumes_effort=False)
|
|
|
|
# Which spelling does the template read? A render that moves on the
|
|
# reasoning_strength kwarg alone marks the muse dialect (its ladder differs).
|
|
strength_dialect = False
|
|
try:
|
|
baseline = render({}, None)
|
|
moved = render({"reasoning_strength": "low"}, None)
|
|
strength_dialect = _renderings_differ(moved, baseline)
|
|
except Exception: # noqa: BLE001 -- a rejecting template is not this dialect
|
|
strength_dialect = False
|
|
|
|
supported = frozenset(name for name in KNOWN_REASONING_EFFORTS if name not in rejected)
|
|
consumes = bool(rejected or diverged or integer_range)
|
|
default = None
|
|
if consumes:
|
|
defaults = [name for name in supported if matches_baseline[name]]
|
|
if defaults:
|
|
default = max(defaults, key=lambda name: EFFORT_SCALE[name])
|
|
return EffortProfile(
|
|
supported=supported,
|
|
default=default,
|
|
consumes_effort=consumes,
|
|
validates=bool(rejected),
|
|
strength_dialect=strength_dialect,
|
|
integer_range=integer_range,
|
|
)
|
|
|
|
|
|
def _accepts_integer_budget(render, base_kwargs, tools) -> bool:
|
|
for value in (0, 101):
|
|
try:
|
|
render({**base_kwargs, "reasoning_effort": value}, tools)
|
|
except Exception: # noqa: BLE001 -- the encoder owns validation
|
|
continue
|
|
return False
|
|
outputs = []
|
|
for value in range(1, 101):
|
|
try:
|
|
outputs.append(render({**base_kwargs, "reasoning_effort": value}, tools))
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
return _renderings_differ(outputs[0], outputs[-1])
|
|
|
|
|
|
def probe_thinking_profile(
|
|
render: Callable[[dict[str, Any], list[dict[str, Any]] | None], Any],
|
|
efforts: EffortProfile,
|
|
) -> ThinkingProfile:
|
|
"""Learn a checkpoint's thinking-toggle behavior by rendering the broadcast
|
|
kwargs through its template. Same contract as ``probe_effort_profile``; a
|
|
template that rejects any toggle probe is treated as not toggleable."""
|
|
try:
|
|
baseline = render({}, None)
|
|
off = render(dict(THINKING_OFF_KWARGS), None)
|
|
on = render(dict(THINKING_ON_KWARGS), None)
|
|
except Exception: # noqa: BLE001 -- can't observe the toggle; assume none
|
|
return ThinkingProfile(efforts=efforts, toggleable=False, has_adaptive=False, default_state="on")
|
|
toggleable = _renderings_differ(off, on)
|
|
has_adaptive = False
|
|
adaptive = None
|
|
if toggleable:
|
|
try:
|
|
adaptive = render(dict(THINKING_ADAPTIVE_KWARGS), None)
|
|
has_adaptive = _renderings_differ(adaptive, off) and _renderings_differ(adaptive, on)
|
|
except Exception: # noqa: BLE001 -- adaptive is not a state this template knows
|
|
has_adaptive = False
|
|
default_state = "on"
|
|
if toggleable:
|
|
if not _renderings_differ(baseline, off):
|
|
default_state = "off"
|
|
elif has_adaptive and not _renderings_differ(baseline, adaptive):
|
|
default_state = "adaptive"
|
|
return ThinkingProfile(
|
|
efforts=efforts,
|
|
toggleable=toggleable,
|
|
has_adaptive=has_adaptive,
|
|
default_state=default_state,
|
|
)
|
|
|
|
|
|
def _renderings_differ(a: Any, b: Any) -> bool:
|
|
try:
|
|
return bool(a != b)
|
|
except Exception: # noqa: BLE001 -- exotic tensor comparison; treat as divergence
|
|
return True
|