Files
FreeToken/tests/server/test_effort_dialect.py
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

261 lines
9.5 KiB
Python

"""Effort/thinking dialect handling at the OpenAI API layer.
Covers the wire-level half of the reasoning-effort pipeline: the superset
validation and DeepSeek ``thinking`` toggle in ``handle_chat_completion``, the
pre-stream render validation, and the probed vocabulary on ``/v1/models``.
The quantization itself is covered in tests/tokenizer/test_effort.py.
"""
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
from fastapi import FastAPI
from fastapi.responses import JSONResponse, StreamingResponse
from fastapi.testclient import TestClient
from freetoken.message import TokenizeMsg, UserReply
from freetoken.server.model_meta import effort_toggle_kwargs
from freetoken.server.openai_api import (
ChatCompletionRequest,
handle_chat_completion,
register_openai_routes,
)
from freetoken.tokenizer.effort import EffortProfile
def run(coro):
return asyncio.run(coro)
class FakeState:
def __init__(self, reasoning_parser: str | None = None) -> None:
self.config = SimpleNamespace(
mm=SimpleNamespace(text_model_only=False, disabled_encoders=frozenset()),
model_path="/models/unit-model",
served_model_name="unit-model",
tool_call_parser="llama3",
reasoning_parser=reasoning_parser,
)
self.sent: TokenizeMsg | None = None
def new_user(self) -> int:
return 42
async def send_one(self, msg):
self.sent = msg
async def wait_for_ack(self, uid: int):
yield UserReply(uid=uid, incremental_output="ok", finished=True, finish_reason="stop")
class FakeManager:
def __init__(self, profile: EffortProfile | None = None, render_error: Exception | None = None):
self._profile = profile
self._render_error = render_error
def effort_profile(self) -> EffortProfile:
assert self._profile is not None
return self._profile
def render_prompt(self, msg) -> str:
if self._render_error is not None:
raise self._render_error
return "rendered"
def chat_request(**overrides) -> ChatCompletionRequest:
payload = {
"model": "unit-model",
"messages": [{"role": "user", "content": "hi"}],
**overrides,
}
return ChatCompletionRequest(**payload)
# --------------------------------------------------------------------------- #
# effort_toggle_kwargs: the DeepSeek thinking toggle folds into template kwargs.
# --------------------------------------------------------------------------- #
OFF = {"enable_thinking": False, "thinking_mode": "disabled"}
ON = {"enable_thinking": True, "thinking_mode": "enabled"}
def test_thinking_disabled_wins_over_an_effort():
ctk = effort_toggle_kwargs("high", {}, thinking_type="disabled")
assert ctk == OFF
def test_thinking_enabled_forwards_the_effort():
ctk = effort_toggle_kwargs("high", {}, thinking_type="enabled")
assert ctk == {**ON, "reasoning_effort": "high"}
def test_thinking_enabled_alone_turns_thinking_on():
ctk = effort_toggle_kwargs(None, {}, thinking_type="enabled")
assert ctk == ON
def test_explicit_template_kwargs_still_win_wholesale():
ctk = effort_toggle_kwargs("high", {"enable_thinking": False}, thinking_type="enabled")
assert ctk == {"enable_thinking": False}
# --------------------------------------------------------------------------- #
# handle_chat_completion: superset validation and the pre-stream render check.
# --------------------------------------------------------------------------- #
def test_unknown_reasoning_effort_is_a_400():
response = run(
handle_chat_completion(chat_request(reasoning_effort="banana"), None, FakeState(), {})
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert "reasoning_effort" in json.loads(response.body)["error"]["message"]
def test_unknown_thinking_type_is_a_400():
response = run(
handle_chat_completion(
chat_request(thinking={"type": "sideways"}), None, FakeState(), {}
)
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
def test_thinking_disabled_reaches_the_tokenizer_as_enable_thinking_false():
state = FakeState(reasoning_parser="qwen3")
response = run(
handle_chat_completion(
chat_request(thinking={"type": "disabled"}), None, state, {}
)
)
assert not isinstance(response, JSONResponse) # plain successful completion
assert state.sent is not None
assert state.sent.chat_template_kwargs == OFF
def test_off_and_mixed_case_efforts_stay_accepted():
# effort_toggle_kwargs has always normalized case/whitespace and honored
# "off" as a disable synonym; the superset gate must not reject them.
for effort, expected in (
("off", OFF),
("High", {**ON, "reasoning_effort": "high"}),
(" high ", {**ON, "reasoning_effort": "high"}),
):
state = FakeState(reasoning_parser="qwen3")
response = run(
handle_chat_completion(chat_request(reasoning_effort=effort), None, state, {})
)
assert not isinstance(response, JSONResponse), effort
assert state.sent.chat_template_kwargs == expected, effort
def test_empty_effort_is_treated_as_absent():
state = FakeState(reasoning_parser="qwen3")
response = run(
handle_chat_completion(chat_request(reasoning_effort=""), None, state, {})
)
assert not isinstance(response, JSONResponse)
assert state.sent.chat_template_kwargs == {}
def test_numeric_effort_rides_template_kwargs_not_the_wire_field():
# The wire field takes named tiers only (as vLLM and SGLang type it); a numeric budget goes
# through chat_template_kwargs, where the checkpoint's encoder validates it at render time.
from pydantic import ValidationError
import pytest
for effort in (37, True, 25.5):
with pytest.raises(ValidationError):
chat_request(reasoning_effort=effort)
ctk = {"enable_thinking": True, "reasoning_effort": 37}
state = FakeState(reasoning_parser="qwen3")
response = run(handle_chat_completion(chat_request(chat_template_kwargs=ctk), None, state, {}))
assert not isinstance(response, JSONResponse)
assert state.sent.chat_template_kwargs == ctk
def test_foreign_thinking_shapes_stay_ignored():
# extra="allow" swallowed any thinking shape before the field existed;
# a bare string, a bool, or a typeless dict must keep working unchanged.
for shape in ("enabled", True, {}, {"budget_tokens": 1024}):
state = FakeState(reasoning_parser="qwen3")
response = run(
handle_chat_completion(chat_request(thinking=shape), None, state, {})
)
assert not isinstance(response, JSONResponse), shape
assert state.sent.chat_template_kwargs == {}, shape
def test_anthropic_style_thinking_dict_works():
state = FakeState(reasoning_parser="qwen3")
response = run(
handle_chat_completion(
chat_request(thinking={"type": "enabled", "budget_tokens": 1024}), None, state, {}
)
)
assert not isinstance(response, JSONResponse)
assert state.sent.chat_template_kwargs == ON
def test_stream_returns_400_when_the_template_rejects_the_render():
state = FakeState()
state.frontend_tokenizer = lambda: FakeManager(
render_error=ValueError("Unexpected reasoning effort high.")
)
response = run(handle_chat_completion(chat_request(stream=True), None, state, {}))
assert isinstance(response, JSONResponse)
assert response.status_code == 400
message = json.loads(response.body)["error"]["message"]
assert message.startswith("could not encode request")
assert state.sent is None # rejected before submission
def test_stream_proceeds_without_a_frontend_tokenizer():
# Minimal embeddings (and the unit FakeState) have no frontend tokenizer;
# validation degrades to the old worker-side path instead of blocking.
response = run(handle_chat_completion(chat_request(stream=True), None, FakeState(), {}))
assert isinstance(response, StreamingResponse)
# --------------------------------------------------------------------------- #
# /v1/models: the probed vocabulary is published; absence stays None.
# --------------------------------------------------------------------------- #
def _models_payload(state) -> dict:
app = FastAPI()
register_openai_routes(app, lambda: state, dict)
with TestClient(app) as client:
response = client.get("/v1/models")
assert response.status_code == 200
return response.json()["data"][0]
def test_v1_models_publishes_the_probed_efforts():
state = FakeState()
state.frontend_tokenizer = lambda: FakeManager(
profile=EffortProfile(
supported=frozenset({"xhigh", "medium", "low"}),
default="xhigh",
consumes_effort=True,
)
)
card = _models_payload(state)
assert card["supported_reasoning_efforts"] == ["xhigh", "medium", "low"]
assert card["default_reasoning_effort"] == "xhigh"
def test_v1_models_omits_efforts_without_a_frontend_tokenizer():
card = _models_payload(FakeState())
assert card["supported_reasoning_efforts"] is None
assert card["default_reasoning_effort"] is None
def test_v1_models_omits_efforts_for_models_without_the_knob():
state = FakeState()
state.frontend_tokenizer = lambda: FakeManager(
profile=EffortProfile(supported=frozenset(), default=None, consumes_effort=False)
)
card = _models_payload(state)
assert card["supported_reasoning_efforts"] is None
assert card["default_reasoning_effort"] is None