mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
313 lines
14 KiB
Python
313 lines
14 KiB
Python
"""Pool-family sizing classmethods: dispatch and parity with the pure cost functions.
|
|
|
|
The engine sizes KV before the pool exists through resolve_pool_class(...).kv_cost /
|
|
solve_num_pages; these pins prove the family dispatch follows the group-spec attn_type
|
|
and that each family's arithmetic is the same pure function the old engine-side policy
|
|
used (byte-identical by delegation)."""
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from freetoken.attention import AttnType
|
|
from freetoken.kvcache import resolve_pool_class
|
|
from freetoken.models.config import KVCacheGroupSpec
|
|
|
|
|
|
def _spec(name, attn_type, *, mla=False, index_head_dim=0, sliding_window=None):
|
|
return KVCacheGroupSpec(
|
|
name=name, layer_ids=(0, 1), num_kv_heads=2, head_dim=64,
|
|
sliding_window=sliding_window, mla=mla, index_head_dim=index_head_dim,
|
|
num_index_layers=2 if index_head_dim else 0, attn_type=attn_type,
|
|
)
|
|
|
|
|
|
def _model_config(specs, **attrs):
|
|
mc = SimpleNamespace(has_swa_attention=False, dsv4_args=None, **attrs)
|
|
mc.kv_cache_group_specs = lambda: specs
|
|
return mc
|
|
|
|
|
|
def test_resolve_pool_class_follows_attn_type():
|
|
from freetoken.kvcache.dsa_pool import DSAKVCache, MLAKVCache
|
|
from freetoken.kvcache.dsv4.v4_pool import DSV4PagedKVCache
|
|
from freetoken.kvcache.hybrid_swa_pool import HybridSWAKVCache
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
cases = [
|
|
((_spec("full", AttnType.FULL),), MHAKVCache),
|
|
((_spec("full", AttnType.FULL), _spec("swa", AttnType.SWA, sliding_window=128)),
|
|
HybridSWAKVCache),
|
|
((_spec("full", AttnType.MLA, mla=True),), MLAKVCache),
|
|
((_spec("full", AttnType.DSA, mla=True, index_head_dim=128),), DSAKVCache),
|
|
((_spec("dsv4", AttnType.DSV4, sliding_window=128),), DSV4PagedKVCache),
|
|
]
|
|
for specs, expected in cases:
|
|
assert resolve_pool_class(_model_config(specs)) is expected
|
|
|
|
# duck-typed configs without the spec walk: dsv4_args marks DSV4, else generic
|
|
assert resolve_pool_class(SimpleNamespace(dsv4_args=object())) is DSV4PagedKVCache
|
|
assert resolve_pool_class(SimpleNamespace()) is MHAKVCache
|
|
|
|
|
|
def _generic_config(num_page_override=None):
|
|
mc = _model_config((_spec("full", AttnType.FULL),))
|
|
mc.linear_attention_group = lambda: None
|
|
return SimpleNamespace(
|
|
model_config=mc, page_size=16, dtype=SimpleNamespace(itemsize=2),
|
|
tp_info=SimpleNamespace(size=1), cache_type="radix",
|
|
swa_full_tokens_ratio=1.0, swa_num_pages_override=None,
|
|
num_page_override=num_page_override, max_running_req=4, max_seq_len=1024,
|
|
)
|
|
|
|
|
|
def test_generic_kv_cost_and_solve_parity():
|
|
from freetoken.kvcache.base import spec_kv_bytes_per_token
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
config = _generic_config()
|
|
(spec,) = config.model_config.kv_cache_group_specs()
|
|
per_page = spec_kv_bytes_per_token(spec, config) * config.page_size
|
|
# 2 slabs x head_dim x kv_heads x itemsize x layers -- the uniform-slab price
|
|
assert per_page == 2 * 64 * 2 * 2 * 2 * config.page_size
|
|
fixed = 0 # uniform slab: no fixed tier
|
|
assert MHAKVCache.kv_cost(config) == (per_page, fixed, config.page_size, 0)
|
|
# solve = (available - fixed) // per_page, override wins verbatim
|
|
assert MHAKVCache.solve_num_pages(config, available_memory=per_page * 100 + fixed) == 100
|
|
assert MHAKVCache.solve_num_pages(_generic_config(num_page_override=7), 0) == 7
|
|
assert MHAKVCache.min_kv_tokens(config) == config.page_size
|
|
|
|
|
|
def _dsv4_config(num_page_override=None):
|
|
from freetoken.models.deepseek_v4.args import DeepseekV4Args
|
|
|
|
args = DeepseekV4Args()
|
|
return SimpleNamespace(
|
|
model_config=SimpleNamespace(dsv4_args=args),
|
|
page_size=args.window_size, max_running_req=2, max_seq_len=4096, cache_type="radix",
|
|
swa_full_tokens_ratio=1.0, swa_num_pages_override=None,
|
|
num_page_override=num_page_override,
|
|
)
|
|
|
|
|
|
def test_dsv4_kv_cost_and_floor_parity():
|
|
from freetoken.kvcache.dsv4.v4_cost_model import _dsv4_swa_ratio, _dsv4_window_floor_pages
|
|
from freetoken.kvcache.dsv4.v4_pool import DSV4PagedKVCache
|
|
|
|
pytest.importorskip("freetoken.kvcache.dsv4.v4_cost_model")
|
|
from freetoken.kvcache.dsv4.v4_cost_model import dsv4_auto_cost_model
|
|
|
|
config = _dsv4_config()
|
|
P = 128
|
|
floor = _dsv4_window_floor_pages(config, P)
|
|
per_page, fixed, min_reserve = dsv4_auto_cost_model(
|
|
config.model_config.dsv4_args, _dsv4_swa_ratio(config), floor, P=P,
|
|
n_scratch=config.max_running_req + 1,
|
|
)
|
|
assert DSV4PagedKVCache.kv_cost(config) == (per_page, fixed, P, min_reserve)
|
|
assert DSV4PagedKVCache.min_kv_tokens(config) == floor * P
|
|
|
|
# an explicit --num-pages below the window working-set floor is a config-time error
|
|
with pytest.raises(ValueError, match="window working-set floor"):
|
|
DSV4PagedKVCache.solve_num_pages(_dsv4_config(num_page_override=1), 0)
|
|
|
|
|
|
def test_startup_kv_budget_composition():
|
|
# The engine-side budget must stay ratio*old - (old - new); a sign/order slip here
|
|
# mis-sizes every model's startup KV pool with the whole CPU suite green.
|
|
from freetoken.engine.engine import _startup_kv_budget
|
|
|
|
assert _startup_kv_budget(0.9, 1000, 400) == int(0.9 * 1000) - 600
|
|
assert _startup_kv_budget(1.0, 1000, 1000) == 1000
|
|
|
|
|
|
def test_generic_validate_rebuild_budget_check():
|
|
from freetoken.kvcache.base import CacheRebuildRejected
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
config = _generic_config()
|
|
per_page, fixed, _, _ = MHAKVCache.kv_cost(config)
|
|
pool = object.__new__(MHAKVCache) # generic validate_rebuild reads no instance state
|
|
budget = per_page * 50 + fixed # memory_ratio=1.0: exactly 50 pages fit
|
|
|
|
def check(pages, baseline):
|
|
pool.validate_rebuild(
|
|
config, num_pages=pages, num_swa_pages=None,
|
|
target_moe=0, per_expert_bytes=0, baseline_free=baseline,
|
|
weights_bytes=0, current_num_pages=10,
|
|
)
|
|
|
|
object.__setattr__(config, "memory_ratio", 1.0)
|
|
check(50, budget) # fits exactly
|
|
with pytest.raises(CacheRebuildRejected, match="old cache kept"):
|
|
check(51, budget)
|
|
# num_pages=None budgets the CURRENT page count
|
|
check(None, per_page * 10 + fixed)
|
|
|
|
|
|
def test_dsv4_validate_rebuild_floor():
|
|
from freetoken.kvcache.base import CacheRebuildRejected
|
|
from freetoken.kvcache.dsv4.v4_cost_model import _dsv4_window_floor_pages
|
|
from freetoken.kvcache.dsv4.v4_pool import DSV4PagedKVCache
|
|
|
|
config = _dsv4_config()
|
|
pool = object.__new__(DSV4PagedKVCache) # floor path reads no instance state
|
|
floor = _dsv4_window_floor_pages(config, config.page_size)
|
|
with pytest.raises(CacheRebuildRejected, match="working-set floor"):
|
|
pool.validate_rebuild(
|
|
config, num_pages=floor - 1, num_swa_pages=None,
|
|
target_moe=0, per_expert_bytes=0, baseline_free=0,
|
|
weights_bytes=0, current_num_pages=100,
|
|
)
|
|
|
|
|
|
def test_rebuild_from_config_explicit_num_swa_pages_wins():
|
|
# The explicit parameter must take precedence over config.swa_num_pages_override --
|
|
# the engine's override write would otherwise mask a dropped parameter forever.
|
|
from freetoken.distributed import set_tp_info, try_get_tp_info
|
|
from freetoken.kvcache.dsv4.v4_cost_model import _dsv4_pool_sizes
|
|
from freetoken.kvcache.hybrid_swa_pool import _swa_paged_num_tokens, _swa_pool_floor
|
|
|
|
if try_get_tp_info() is None:
|
|
set_tp_info(rank=0, size=1)
|
|
|
|
swa_cfg = _generic_config()
|
|
swa_specs = (
|
|
_spec("full", AttnType.FULL),
|
|
_spec("swa", AttnType.SWA, sliding_window=32),
|
|
)
|
|
swa_cfg.model_config.kv_cache_group_specs = lambda: swa_specs
|
|
object.__setattr__(swa_cfg, "swa_num_pages_override", 999_999)
|
|
floor = _swa_pool_floor(swa_cfg)
|
|
explicit = 2 * floor # above the floor so the pin is what we read back
|
|
assert _swa_paged_num_tokens(swa_cfg, 11, num_swa_pages=explicit) == explicit + 1
|
|
assert _swa_paged_num_tokens(swa_cfg, 11) == 999_999 + 1 # override path unchanged
|
|
|
|
from freetoken.kvcache.dsv4.v4_cost_model import _dsv4_window_floor_pages
|
|
|
|
d_cfg = _dsv4_config()
|
|
d_floor = _dsv4_window_floor_pages(d_cfg, d_cfg.page_size)
|
|
pin = d_floor + 3 # above the floor so the pin (not the floor) is what we read back
|
|
a = _dsv4_pool_sizes(d_cfg, pin + 40, num_swa_pages=pin)
|
|
object.__setattr__(d_cfg, "swa_num_pages_override", 4)
|
|
b = _dsv4_pool_sizes(d_cfg, pin + 40, num_swa_pages=pin)
|
|
# +1 = the sentinel window page (same convention as _swa_paged_num_tokens's +1 slot)
|
|
assert a.n_win_pages == b.n_win_pages == pin + 1 # explicit target wins over the override
|
|
|
|
|
|
def test_create_kv_pool_builds_the_right_family():
|
|
import torch
|
|
|
|
from freetoken.distributed import set_tp_info, try_get_tp_info
|
|
from freetoken.kvcache import create_kv_pool
|
|
from freetoken.kvcache.hybrid_swa_pool import _swa_paged_num_tokens
|
|
from freetoken.kvcache.hybrid_swa_pool import HybridSWAKVCache
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
if try_get_tp_info() is None:
|
|
set_tp_info(rank=0, size=1)
|
|
|
|
mc = SimpleNamespace(
|
|
has_swa_attention=False, has_linear_attention=False, dsv4_args=None,
|
|
num_layers=2, num_kv_heads=1, head_dim=8,
|
|
)
|
|
mc.kv_cache_group_specs = lambda: (_spec("full", AttnType.FULL),)
|
|
config = SimpleNamespace(
|
|
model_config=mc, page_size=1, cache_type="radix", max_running_req=2,
|
|
swa_full_tokens_ratio=1.0, swa_num_pages_override=None, max_seq_len=64,
|
|
)
|
|
pool = create_kv_pool(config, num_pages=8, device=torch.device("cpu"), dtype=torch.bfloat16)
|
|
assert isinstance(pool, MHAKVCache)
|
|
|
|
import dataclasses
|
|
|
|
swa_specs = (
|
|
dataclasses.replace(_spec("full", AttnType.FULL), layer_ids=(0,)),
|
|
dataclasses.replace(_spec("swa", AttnType.SWA, sliding_window=4), layer_ids=(1,)),
|
|
)
|
|
mc2 = SimpleNamespace(
|
|
has_swa_attention=True, has_linear_attention=False, dsv4_args=None, num_layers=2,
|
|
)
|
|
mc2.kv_cache_group_specs = lambda: swa_specs
|
|
config2 = SimpleNamespace(
|
|
model_config=mc2, page_size=1, cache_type="swa_radix", max_running_req=2,
|
|
swa_full_tokens_ratio=0.5, swa_num_pages_override=None, max_seq_len=64,
|
|
max_extend_tokens=64,
|
|
)
|
|
pool2 = create_kv_pool(config2, num_pages=8, device=torch.device("cpu"), dtype=torch.bfloat16)
|
|
assert isinstance(pool2, HybridSWAKVCache)
|
|
assert pool2.swa_num_tokens == _swa_paged_num_tokens(config2, 9)
|
|
|
|
|
|
def test_linear_state_pool_prices_itself():
|
|
# The GDN state pool is a sibling pool: the KV family's kv_cost excludes it and the
|
|
# engine adds state_pool_bytes -- the sum must equal the old single-walk total.
|
|
from freetoken.kvcache.linear_state_pool import (
|
|
_linear_pool_num_slots, linear_state_bytes_per_req, state_pool_bytes,
|
|
)
|
|
from freetoken.models.config import LinearGatedDeltaGroupConfig
|
|
|
|
group = LinearGatedDeltaGroupConfig(
|
|
name="linear", layer_ids=(1, 3), num_key_heads=2, num_value_heads=4,
|
|
key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",
|
|
)
|
|
config = _generic_config()
|
|
config.linear_state_cache_ratio = 0.5
|
|
config.cache_type = "hybrid_radix"
|
|
config.model_config.linear_attention_group = lambda: group
|
|
per_req = linear_state_bytes_per_req(group, config.tp_info.size, config.dtype)
|
|
assert state_pool_bytes(config) == per_req * _linear_pool_num_slots(config)
|
|
assert state_pool_bytes(config, num_slots=7) == per_req * 7
|
|
# models without a linear group price to zero
|
|
config.model_config.linear_attention_group = lambda: None
|
|
assert state_pool_bytes(config) == 0
|
|
|
|
|
|
def test_validate_rebuild_targets_flow_by_kv_cost_signature():
|
|
"""The base template hands each family's kv_cost exactly the non-None target keys its
|
|
signature declares: hybrid's pinned window flows through and changes the verdict;
|
|
uniform pools never see keys they don't declare (no TypeError on the same call)."""
|
|
from freetoken.distributed import set_tp_info, try_get_tp_info
|
|
from freetoken.kvcache.base import CacheRebuildRejected
|
|
from freetoken.kvcache.hybrid_swa_pool import HybridSWAKVCache, _swa_pool_floor
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
if try_get_tp_info() is None:
|
|
set_tp_info(rank=0, size=1)
|
|
|
|
config = _generic_config()
|
|
specs = (
|
|
_spec("full", AttnType.FULL),
|
|
_spec("swa", AttnType.SWA, sliding_window=32),
|
|
)
|
|
config.model_config.kv_cache_group_specs = lambda: specs
|
|
object.__setattr__(config, "cache_type", "swa_radix")
|
|
object.__setattr__(config, "memory_ratio", 1.0)
|
|
|
|
def check(cls, budget_tokens_kwargs, **targets):
|
|
pool = object.__new__(cls)
|
|
per_page, fixed, _, _ = cls.kv_cost(config, **budget_tokens_kwargs)
|
|
pool.validate_rebuild(
|
|
config, num_pages=10, target_moe=0, per_expert_bytes=0,
|
|
baseline_free=10 * per_page + fixed, weights_bytes=0,
|
|
current_num_pages=10, **targets,
|
|
)
|
|
|
|
# exactly-fitting budget passes with the pin priced in ...
|
|
pin = 2 * _swa_pool_floor(config)
|
|
check(HybridSWAKVCache, {"num_swa_pages": pin}, num_swa_pages=pin)
|
|
# ... and the same budget WITHOUT the pin priced in rejects: the pin reached kv_cost
|
|
with pytest.raises(CacheRebuildRejected):
|
|
check(HybridSWAKVCache, {}, num_swa_pages=pin)
|
|
|
|
# uniform pool: stray family keys are filtered, never a TypeError
|
|
plain = _generic_config()
|
|
object.__setattr__(plain, "memory_ratio", 1.0)
|
|
per_page = MHAKVCache.kv_cost(plain)[0]
|
|
pool = object.__new__(MHAKVCache)
|
|
pool.validate_rebuild(
|
|
plain, num_pages=10, target_moe=0, per_expert_bytes=0,
|
|
baseline_free=10 * per_page, weights_bytes=0, current_num_pages=10,
|
|
num_swa_pages=None, future_family_key=None,
|
|
)
|