mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
374 lines
13 KiB
Python
374 lines
13 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
|
||
from freetoken.server.function_call_parser import FunctionCallParser, SUPPORTED_TOOL_CALL_PARSERS
|
||
|
||
|
||
TOOLS = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "get_weather",
|
||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||
},
|
||
}
|
||
]
|
||
|
||
OPENCODE_TOOLS = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "read",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {"filePath": {"type": "string"}},
|
||
},
|
||
},
|
||
},
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "glob",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"pattern": {"type": "string"},
|
||
"path": {"type": "string"},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
]
|
||
|
||
|
||
@pytest.mark.parametrize("parser_name", SUPPORTED_TOOL_CALL_PARSERS)
|
||
def test_supported_parser_names_instantiate(parser_name):
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser=parser_name)
|
||
|
||
result = parser.parse_non_stream("plain text")
|
||
|
||
assert result.normal_text == "plain text"
|
||
assert result.calls == []
|
||
|
||
|
||
def test_mistral_parser_accepts_tool_calls_tag():
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="mistral")
|
||
|
||
result = parser.parse_non_stream(
|
||
'[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]'
|
||
)
|
||
|
||
assert result.normal_text == ""
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "get_weather"
|
||
assert json.loads(result.calls[0].parameters) == {"city": "Paris"}
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("parser_name", "text"),
|
||
[
|
||
(
|
||
"mistral",
|
||
'[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]',
|
||
),
|
||
(
|
||
"qwen25",
|
||
'<tool_call>{"name":"get_weather","arguments":{"city":"Paris"}}</tool_call>',
|
||
),
|
||
(
|
||
"llama3",
|
||
'<|python_tag|>{"name":"get_weather","arguments":{"city":"Paris"}}',
|
||
),
|
||
],
|
||
)
|
||
def test_base_json_parsers_use_declared_tool_index(parser_name, text):
|
||
tools = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "other_tool",
|
||
"parameters": {"type": "object"},
|
||
},
|
||
},
|
||
TOOLS[0],
|
||
]
|
||
parser = FunctionCallParser(tools, tool_call_parser=parser_name)
|
||
|
||
result = parser.parse_non_stream(text)
|
||
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "get_weather"
|
||
assert result.calls[0].tool_index == 1
|
||
|
||
|
||
def test_base_json_parsers_forward_unknown_tools_by_default():
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="mistral")
|
||
|
||
result = parser.parse_non_stream(
|
||
'[TOOL_CALLS] [{"name":"not_declared","arguments":{"city":"Paris"}}]'
|
||
)
|
||
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].tool_index == -1
|
||
assert result.calls[0].name == "not_declared"
|
||
assert json.loads(result.calls[0].parameters) == {"city": "Paris"}
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"text",
|
||
[
|
||
'<tool_call>{"name":"get_weather","arguments":{"city":"Paris"}}</tool_call>',
|
||
],
|
||
)
|
||
def test_parser_accepts_common_tagged_tool_call_shapes(text):
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="qwen25")
|
||
|
||
result = parser.parse_non_stream(text)
|
||
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "get_weather"
|
||
assert json.loads(result.calls[0].parameters) == {"city": "Paris"}
|
||
|
||
|
||
def test_gemma4_parser_accepts_compact_tool_call_shape():
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="gemma4")
|
||
|
||
result = parser.parse_non_stream('<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}<tool_call|>')
|
||
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "get_weather"
|
||
assert json.loads(result.calls[0].parameters) == {"city": "Paris"}
|
||
|
||
|
||
def test_gemma4_parser_accepts_namespaced_tool_name():
|
||
tools = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "superpowers:using_superpowers",
|
||
"parameters": {"type": "object"},
|
||
},
|
||
}
|
||
]
|
||
parser = FunctionCallParser(tools, tool_call_parser="gemma4")
|
||
|
||
result = parser.parse_non_stream(
|
||
"<|tool_call>call:superpowers:using_superpowers{}<tool_call|>"
|
||
)
|
||
|
||
assert result.normal_text == ""
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "superpowers:using_superpowers"
|
||
assert json.loads(result.calls[0].parameters) == {}
|
||
|
||
|
||
def test_gemma4_parser_forwards_namespaced_skill_without_declared_tool():
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="gemma4")
|
||
|
||
result = parser.parse_non_stream(
|
||
"<|tool_call>call:superpowers:using_superpowers{}<tool_call|>"
|
||
)
|
||
|
||
assert result.normal_text == ""
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "superpowers:using_superpowers"
|
||
assert json.loads(result.calls[0].parameters) == {}
|
||
|
||
|
||
def test_gpt_oss_parser_accepts_namespaced_tool_name():
|
||
tools = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "superpowers:using_superpowers",
|
||
"parameters": {"type": "object"},
|
||
},
|
||
}
|
||
]
|
||
parser = FunctionCallParser(tools, tool_call_parser="gpt_oss")
|
||
|
||
result = parser.parse_non_stream(
|
||
"<|start|>assistant<|channel|>commentary "
|
||
"to=functions.superpowers:using_superpowers <|constrain|>json<|message|>{}<|end|>"
|
||
)
|
||
|
||
assert result.normal_text == ""
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == "superpowers:using_superpowers"
|
||
assert json.loads(result.calls[0].parameters) == {}
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("parser_name", "text", "expected_name", "expected_args"),
|
||
[
|
||
(
|
||
"qwen3_coder",
|
||
"<tool_call><function=read><parameter=filePath>/tmp/test_calc.py</parameter></function></tool_call>",
|
||
"read",
|
||
{"filePath": "/tmp/test_calc.py"},
|
||
),
|
||
(
|
||
"glm47",
|
||
"<tool_call>read<arg_key>filePath</arg_key><arg_value>/tmp/test_calc.py</arg_value></tool_call>",
|
||
"read",
|
||
{"filePath": "/tmp/test_calc.py"},
|
||
),
|
||
(
|
||
"minimax",
|
||
"<minimax:tool_call><invoke name=\"read\"><parameter name=\"filePath\">"
|
||
"/tmp/test_calc.py</parameter></invoke></minimax:tool_call>",
|
||
"read",
|
||
{"filePath": "/tmp/test_calc.py"},
|
||
),
|
||
(
|
||
"gpt_oss",
|
||
"<|channel|>analysis<|message|>Need files.<|end|><|start|>assistant"
|
||
"<|channel|>commentary to=functions.glob <|constrain|>json<|message|>"
|
||
"{\"pattern\":\"**/*.py\",\"path\":\"/tmp/ws\"}",
|
||
"glob",
|
||
{"pattern": "**/*.py", "path": "/tmp/ws"},
|
||
),
|
||
(
|
||
"deepseekv32",
|
||
"<|DSML|function_calls><|DSML|invoke name=\"read\">"
|
||
"<|DSML|parameter name=\"filePath\" string=\"true\">/tmp/test_calc.py</|DSML|parameter>"
|
||
"</|DSML|invoke></|DSML|function_calls>",
|
||
"read",
|
||
{"filePath": "/tmp/test_calc.py"},
|
||
),
|
||
(
|
||
"deepseekv32",
|
||
"<|DSML|tool_calls><|DSML|invoke name=\"read\">"
|
||
"<|DSML|parameter name=\"filePath\">/tmp/test_calc.py</|DSML|parameter>"
|
||
"</|DSML|invoke></|DSML|tool_calls>",
|
||
"read",
|
||
{"filePath": "/tmp/test_calc.py"},
|
||
),
|
||
],
|
||
)
|
||
def test_parser_accepts_family_specific_tool_call_shapes(
|
||
parser_name, text, expected_name, expected_args
|
||
):
|
||
parser = FunctionCallParser(OPENCODE_TOOLS, tool_call_parser=parser_name)
|
||
|
||
result = parser.parse_non_stream(text)
|
||
|
||
assert result.normal_text in ("", "Need files.")
|
||
assert len(result.calls) == 1
|
||
assert result.calls[0].name == expected_name
|
||
assert json.loads(result.calls[0].parameters) == expected_args
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Streaming incremental parsing (parse_stream_chunk)
|
||
# --------------------------------------------------------------------------- #
|
||
def _feed(parser, chunks):
|
||
"""Feed chunks through the streaming parser; return (per-chunk normal texts, calls)."""
|
||
texts, calls = [], []
|
||
for chunk in chunks:
|
||
normal, chunk_calls = parser.parse_stream_chunk(chunk)
|
||
texts.append(normal)
|
||
calls.extend(chunk_calls)
|
||
return texts, calls
|
||
|
||
|
||
@pytest.mark.parametrize("parser_name", ["qwen25", "glm47", "gemma4", "minimax", "deepseekv32", "qwen3_coder"])
|
||
def test_streaming_plain_text_releases_per_chunk(parser_name):
|
||
# A pure-text response must stream out chunk by chunk, not buffer to the end.
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser=parser_name)
|
||
texts, calls = _feed(parser, ["Hello ", "world."])
|
||
assert texts == ["Hello ", "world."]
|
||
assert calls == []
|
||
assert parser.finish_stream() == ""
|
||
|
||
|
||
def test_streaming_partial_tag_holdback_then_release():
|
||
# Text held back as a suspected tag prefix must be released once disambiguated.
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="qwen25")
|
||
texts, _ = _feed(parser, ["Hi <", "there"])
|
||
assert "".join(texts) + parser.finish_stream() == "Hi <there"
|
||
|
||
|
||
def test_dsv32_streaming_partial_prefix_not_dropped():
|
||
# Regression: the non-tool branch used to clear the whole buffer but return only
|
||
# the newest chunk, dropping text held back as a suspected partial bot_token.
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="deepseekv32")
|
||
texts, calls = _feed(parser, ["a<", "b"])
|
||
assert "".join(texts) + parser.finish_stream() == "a<b"
|
||
assert calls == []
|
||
|
||
|
||
def test_streaming_finish_stream_releases_held_tail():
|
||
parser = FunctionCallParser(TOOLS, tool_call_parser="qwen25")
|
||
texts, calls = _feed(parser, ["text <tool"])
|
||
assert calls == []
|
||
assert "".join(texts) + parser.finish_stream() == "text <tool"
|
||
|
||
|
||
def test_dsv32_streaming_multi_param_args_prefix_stable():
|
||
# The DSML streaming state machine emits prefix-stable fragments (vLLM-style):
|
||
# '{"key":"' at parameter open, escaped value chars, '"' at close, '}' at
|
||
# invoke close — concatenation IS the final arguments JSON.
|
||
parser = FunctionCallParser(OPENCODE_TOOLS, tool_call_parser="deepseekv32")
|
||
assert parser.args_fragments_prefix_stable() is True
|
||
block = (
|
||
"<|DSML|function_calls>\n"
|
||
'<|DSML|invoke name="glob">\n'
|
||
'<|DSML|parameter name="pattern" string="true">*.py</|DSML|parameter>\n'
|
||
'<|DSML|parameter name="path" string="true">/src</|DSML|parameter>\n'
|
||
"</|DSML|invoke>\n"
|
||
"</|DSML|function_calls>"
|
||
)
|
||
chunks = [block[i : i + 7] for i in range(0, len(block), 7)]
|
||
_, calls = _feed(parser, chunks)
|
||
named = [c for c in calls if c.name]
|
||
assert len(named) == 1 and named[0].name == "glob"
|
||
joined = "".join(c.parameters for c in calls if c.name is None)
|
||
assert json.loads(joined) == {"pattern": "*.py", "path": "/src"}
|
||
# multiple argument fragments streamed mid-call, not one blob at close
|
||
assert sum(1 for c in calls if c.name is None) >= 4
|
||
# detector parse state agrees (used as the truncation fallback)
|
||
assert json.loads(parser.unstreamed_arguments(named[0].tool_index)) == {
|
||
"pattern": "*.py",
|
||
"path": "/src",
|
||
}
|
||
|
||
|
||
V41_BLOCK = (
|
||
"<|DSML| calls>\n"
|
||
'<|DSML| invoke name="glob">\n'
|
||
'<|DSML| parameter name="pattern" string="true">*.py</|DSML| parameter>\n'
|
||
'<|DSML| parameter name="path" string="true">/src</|DSML| parameter>\n'
|
||
"</|DSML| invoke>\n"
|
||
"</|DSML| calls>"
|
||
)
|
||
|
||
|
||
def test_dsv41_dsml_tags_lead_with_a_space():
|
||
"""DeepSeek-V4.1 leads every DSML tag name with a space (its encoder's ``<|DSML| calls>``):
|
||
the V3.2 detector sees no tool call there, the V4.1 one parses it whole and streamed."""
|
||
text = "Let me look.\n\n" + V41_BLOCK
|
||
assert not FunctionCallParser(OPENCODE_TOOLS, tool_call_parser="deepseekv32").has_tool_call(text)
|
||
parser = FunctionCallParser(OPENCODE_TOOLS, tool_call_parser="deepseekv41")
|
||
assert parser.has_tool_call(text)
|
||
result = parser.parse_non_stream(text)
|
||
assert result.normal_text == "Let me look."
|
||
assert [(c.name, json.loads(c.parameters)) for c in result.calls] == [("glob", {"pattern": "*.py", "path": "/src"})]
|
||
|
||
streamed = FunctionCallParser(OPENCODE_TOOLS, tool_call_parser="deepseekv41")
|
||
texts, calls = _feed(streamed, [text[i : i + 5] for i in range(0, len(text), 5)])
|
||
assert "".join(texts).strip() + streamed.finish_stream() == "Let me look."
|
||
assert [c.name for c in calls if c.name] == ["glob"]
|
||
assert json.loads("".join(c.parameters for c in calls if c.name is None)) == {"pattern": "*.py", "path": "/src"}
|
||
|
||
|
||
def test_streaming_support_flags():
|
||
# Every registered detector is incremental-safe (buffered fallback remains as
|
||
# the escape hatch for future formats, exercised via monkeypatch in
|
||
# test_streaming_model_matrix.py::test_non_streaming_detector_falls_back_to_buffered_parse).
|
||
for name in SUPPORTED_TOOL_CALL_PARSERS:
|
||
assert FunctionCallParser(TOOLS, tool_call_parser=name).supports_streaming() is True
|