mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor: add engine and scheduler seams for multi-row decode steps * refactor: rename the graph steps argument to rows_per_req * refactor: drop VerifyInfo and require rows_per_req on verify batches * refactor: fix stale seam comments and read decode_rows_per_req directly * refactor: reword two GraphRunner comments for multi-row graphs * fix(engine): scale the GPU-decode MoE pool limits by decode_rows_per_req * refactor: drop the unread Context.moe_offload_cache field * test: trim the multi-row seam tests
583 lines
22 KiB
Python
583 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
import os
|
|
|
|
from freetoken.engine.cache_budget import expert_bytes_per_slot, plan_cache_budget, resolve_moe_cache_auto
|
|
from freetoken.engine.engine import _pin_budget_bytes
|
|
|
|
|
|
def test_moe_priority_fills_experts_up_to_total():
|
|
# budget large enough to cache every expert; KV gets the remainder.
|
|
# per_expert=100, cache_per_page=10, total=8 experts (L*E), E=4.
|
|
size, pages, overlap = plan_cache_budget(
|
|
budget_bytes=2000, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=4, total_experts=8, prefill_overlap=True,
|
|
kv_reserve_pages=5, max_slots=8,
|
|
)
|
|
assert size == 8 # capped at full residency
|
|
assert pages == (2000 - 8 * 100) // 10 # == 120, remainder to KV
|
|
assert overlap is True
|
|
|
|
|
|
def test_offload_case_experts_take_most_kv_gets_reserve_floor():
|
|
# budget too small for full residency: experts take what they can, KV keeps its floor.
|
|
size, pages, overlap = plan_cache_budget(
|
|
budget_bytes=1000, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=2, total_experts=50, prefill_overlap=True,
|
|
kv_reserve_pages=10, max_slots=50,
|
|
)
|
|
# raw = (1000 - 10*10) // 100 = 9 ; clamped to [4, 50] -> 9
|
|
assert size == 9
|
|
assert pages == max((1000 - 9 * 100) // 10, 10) # remainder 10 pages, == floor
|
|
assert overlap is True
|
|
|
|
|
|
def test_marlin_cap_clamps_count_and_rolls_bytes_to_kv():
|
|
# budget would fund 1500 experts, but marlin caps at 992; freed bytes become KV pages.
|
|
size, pages, overlap = plan_cache_budget(
|
|
budget_bytes=200_000, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=128, total_experts=4000, prefill_overlap=True,
|
|
kv_reserve_pages=0, max_slots=992,
|
|
)
|
|
assert size == 992
|
|
assert pages == (200_000 - 992 * 100) // 10
|
|
|
|
|
|
def test_small_cache_disables_prefill_overlap():
|
|
# cap below 2*num_experts -> overlap impossible, falls back to num_experts floor.
|
|
size, pages, overlap = plan_cache_budget(
|
|
budget_bytes=10_000, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=8, total_experts=12, prefill_overlap=True,
|
|
kv_reserve_pages=0, max_slots=12,
|
|
)
|
|
assert overlap is False
|
|
# raw = 10000//100 = 100, clamped to hi = min(12, 12) = 12.
|
|
assert size == 12
|
|
|
|
|
|
def test_insufficient_kv_memory_raises():
|
|
with pytest.raises(AssertionError, match="not enough memory"):
|
|
plan_cache_budget(
|
|
budget_bytes=410, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=4, total_experts=4, prefill_overlap=False,
|
|
kv_reserve_pages=0, max_slots=4,
|
|
) # experts eat 400, KV gets 1 page -> not > 1
|
|
|
|
|
|
def test_budget_too_small_for_min_moe_plus_reserve_raises():
|
|
# Budget cannot fund even the minimum MoE slots + the KV reserve, so the floored plan
|
|
# would exceed budget_bytes. Reject in arithmetic rather than OOM in a later CUDA alloc.
|
|
with pytest.raises(AssertionError, match="budget too small"):
|
|
plan_cache_budget(
|
|
budget_bytes=300, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=4, total_experts=4, prefill_overlap=False,
|
|
kv_reserve_pages=10, max_slots=4,
|
|
) # min moe = 4 slots (400 B) + reserve (10 pages = 100 B) = 500 B > 300 B budget
|
|
|
|
|
|
def test_prefill_overlap_false_is_honored():
|
|
# Even when the cache could fit 2*num_experts, an explicit False stays False.
|
|
size, pages, overlap = plan_cache_budget(
|
|
budget_bytes=2000, per_expert_bytes=100, cache_per_page=10,
|
|
num_experts=4, total_experts=8, prefill_overlap=False,
|
|
kv_reserve_pages=0, max_slots=8,
|
|
)
|
|
assert size == 8
|
|
assert overlap is False
|
|
assert pages == (2000 - 8 * 100) // 10
|
|
|
|
|
|
def test_expert_bytes_per_slot_sums_row_bytes_over_banks():
|
|
sources = {
|
|
"gate_up": [torch.zeros(4, 32, 8, dtype=torch.float16)], # row = 32*8*2 = 512
|
|
"down": [torch.zeros(4, 8, 16, dtype=torch.float16)], # row = 8*16*2 = 256
|
|
}
|
|
assert expert_bytes_per_slot(sources) == 512 + 256
|
|
|
|
|
|
def test_resolve_auto_applies_ratio_once():
|
|
# baseline 1000, weights 100, ratio 0.9 -> budget = 900 - 100 - 0(fixed) = 800
|
|
size, pages, overlap = resolve_moe_cache_auto(
|
|
baseline_free=1000, weights_bytes=100, memory_ratio=0.9,
|
|
cache_per_page=10, fixed_cache_size=0, per_expert_bytes=50,
|
|
num_experts=4, total_experts=8, prefill_overlap=True,
|
|
kv_reserve_tokens=0, page_size=1,
|
|
)
|
|
# budget 800: experts cap at 8 -> 400 bytes; KV = 400//10 = 40 pages
|
|
assert size == 8 and pages == 40 and overlap is True
|
|
|
|
|
|
def test_resolve_auto_caps_slots_at_the_kernel_limit():
|
|
size, _, _ = resolve_moe_cache_auto(
|
|
baseline_free=10_000_000, weights_bytes=0, memory_ratio=1.0,
|
|
cache_per_page=10, fixed_cache_size=0, per_expert_bytes=100,
|
|
num_experts=128, total_experts=4000, prefill_overlap=False,
|
|
kv_reserve_tokens=0, page_size=1, max_slots=992,
|
|
)
|
|
assert size == 992
|
|
|
|
|
|
def _dsv4_adjust_cfg(**over):
|
|
# A DSV4 _adjust_config stub mirroring the real checkpoint (ds_fp4 experts, dsv4_sparse
|
|
# attention, offload MoE backend).
|
|
from types import SimpleNamespace
|
|
|
|
model_config = SimpleNamespace(
|
|
single_stream_only=False, dsv4_args=SimpleNamespace(window_size=128), is_moe=True,
|
|
expert_quant="ds_fp4", has_swa_attention=False, has_linear_attention=False,
|
|
)
|
|
|
|
class Cfg:
|
|
moe_cache_auto = True
|
|
moe_cache_size = 0
|
|
moe_cache_rate = None
|
|
moe_strategy = "offload"
|
|
max_running_req = 1
|
|
cuda_graph_max_bs = 1
|
|
cuda_graph_bs = [1]
|
|
max_seq_len = 1024
|
|
max_extend_tokens = 4096
|
|
page_size = 1
|
|
attention_backend = "dsv4_sparse"
|
|
moe_cpu_layers = None
|
|
num_page_override = None
|
|
num_token_override = None
|
|
|
|
@property
|
|
def model_config(self):
|
|
return model_config
|
|
|
|
cfg = Cfg()
|
|
for k, v in over.items():
|
|
object.__setattr__(cfg, k, v)
|
|
return cfg
|
|
|
|
|
|
def test_adjust_config_allows_auto_for_dsv4():
|
|
# DSV4 now supports --moe-cache-auto via the affine KV cost bridge (dsv4_auto_cost_model);
|
|
# _adjust_config must NOT reject it.
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
cfg = _dsv4_adjust_cfg()
|
|
_adjust_config(cfg) # must not raise
|
|
assert cfg.moe_strategy == "offload"
|
|
assert cfg.moe_cache_auto is True # resolved later at engine init, not here
|
|
assert cfg.page_size == 128 # DSV4's KV page is the P-token window page
|
|
|
|
|
|
def test_adjust_config_resolves_num_tokens_for_dsv4():
|
|
# --num-tokens resolves AFTER every page_size override, so DSV4's P=128 page divides it.
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
cfg = _dsv4_adjust_cfg(num_token_override=131072)
|
|
_adjust_config(cfg)
|
|
assert cfg.page_size == 128
|
|
assert cfg.num_page_override == 1024
|
|
|
|
|
|
def test_adjust_config_rejects_num_tokens_not_multiple_of_page():
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
cfg = _dsv4_adjust_cfg(num_token_override=131000) # not a multiple of 128
|
|
with pytest.raises(ValueError, match="not a multiple"):
|
|
_adjust_config(cfg)
|
|
|
|
|
|
def test_adjust_config_rejects_num_tokens_with_num_pages():
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
cfg = _dsv4_adjust_cfg(num_token_override=131072, num_page_override=1024)
|
|
with pytest.raises(ValueError, match="mutually exclusive"):
|
|
_adjust_config(cfg)
|
|
|
|
|
|
def test_adjust_config_resolves_num_tokens_generic():
|
|
# Generic model keeps its page_size (1 here): tokens map 1:1 onto pages.
|
|
from types import SimpleNamespace
|
|
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
model_config = SimpleNamespace(
|
|
single_stream_only=False, is_moe=False, expert_quant="none",
|
|
has_swa_attention=False, has_linear_attention=False,
|
|
)
|
|
|
|
class Cfg:
|
|
moe_cache_auto = False
|
|
moe_cache_size = 0
|
|
moe_cache_rate = None
|
|
moe_strategy = "auto"
|
|
max_running_req = 4
|
|
cuda_graph_max_bs = 2
|
|
cuda_graph_bs = [1, 2]
|
|
max_seq_len = 1024
|
|
page_size = 1
|
|
attention_backend = "triton" if torch.version.hip is not None else "fi"
|
|
num_page_override = None
|
|
num_token_override = 5000
|
|
|
|
@property
|
|
def model_config(self):
|
|
return model_config
|
|
|
|
cfg = Cfg()
|
|
_adjust_config(cfg)
|
|
assert cfg.num_page_override == 5000
|
|
|
|
|
|
def test_mha_kv_cost_simple_full_attention():
|
|
import torch
|
|
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
from freetoken.models.config import KVCacheGroupSpec
|
|
from freetoken.utils import div_even
|
|
|
|
class StubModelConfig:
|
|
has_swa_attention = False
|
|
|
|
def kv_cache_group_specs(self):
|
|
return [KVCacheGroupSpec(
|
|
name="full", layer_ids=tuple(range(3)),
|
|
num_kv_heads=8, head_dim=64, sliding_window=None,
|
|
)]
|
|
|
|
def linear_attention_group(self):
|
|
return None
|
|
|
|
class StubConfig:
|
|
dtype = torch.bfloat16
|
|
page_size = 16
|
|
max_running_req = 4
|
|
swa_full_tokens_ratio = 0.2
|
|
swa_num_pages_override = None
|
|
model_config = StubModelConfig()
|
|
|
|
class tp_info:
|
|
size = 1
|
|
|
|
cache_per_page, fixed, _, _ = MHAKVCache.kv_cost(StubConfig())
|
|
per_token = 2 * 64 * div_even(8, 1, allow_replicate=True) * 2 * 3
|
|
assert cache_per_page == per_token * 16
|
|
assert fixed == 0
|
|
|
|
|
|
def test_engine_resolve_auto_moe_cache_size_maps_kwargs():
|
|
import torch
|
|
|
|
from freetoken.engine.engine import Engine
|
|
from freetoken.models.config import KVCacheGroupSpec
|
|
|
|
class StubModelConfig:
|
|
has_swa_attention = False
|
|
num_experts = 4
|
|
num_moe_layers = 2 # total_experts = 8
|
|
|
|
def kv_cache_group_specs(self):
|
|
return [KVCacheGroupSpec(
|
|
name="full", layer_ids=(0, 1, 2), num_kv_heads=8, head_dim=64, sliding_window=None,
|
|
)]
|
|
|
|
def linear_attention_group(self):
|
|
return None
|
|
|
|
class StubConfig:
|
|
dtype = torch.float16
|
|
page_size = 16
|
|
max_running_req = 4
|
|
hybrid_swa_cache_mode = "auto"
|
|
memory_ratio = 0.9
|
|
moe_prefill_overlap = True
|
|
kv_reserve_tokens = 0
|
|
swa_full_tokens_ratio = 0.2
|
|
swa_num_pages_override = None
|
|
model_config = StubModelConfig()
|
|
|
|
class tp_info:
|
|
size = 1
|
|
|
|
class StubBanks:
|
|
# 2 layers (num_moe_layers above) x 4 experts each -- per-layer host bank contract.
|
|
sources = {
|
|
"gate_up": [torch.zeros(4, 32, 8, dtype=torch.float16)] * 2, # row = 32*8*2 = 512
|
|
"down": [torch.zeros(4, 8, 16, dtype=torch.float16)] * 2, # row = 8*16*2 = 256
|
|
}
|
|
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
engine = Engine.__new__(Engine) # bypass __init__/GPU
|
|
engine._baseline_free = 10_000_000
|
|
engine._weights_bytes = 1_000_000
|
|
engine._pool_cls = MHAKVCache # __init__ skipped -> install the generic pool family
|
|
|
|
size, pages, overlap = engine._resolve_auto_moe_cache_size(StubConfig(), StubBanks())
|
|
|
|
# cross-check against the same pure functions, proving the kwarg mapping is faithful
|
|
from freetoken.engine.cache_budget import expert_bytes_per_slot, resolve_moe_cache_auto
|
|
from freetoken.kvcache.mha_pool import MHAKVCache
|
|
|
|
cache_per_page, fixed, _, _ = MHAKVCache.kv_cost(StubConfig())
|
|
expected = resolve_moe_cache_auto(
|
|
baseline_free=10_000_000, weights_bytes=1_000_000, memory_ratio=0.9,
|
|
cache_per_page=cache_per_page, fixed_cache_size=fixed,
|
|
per_expert_bytes=expert_bytes_per_slot(StubBanks.sources),
|
|
num_experts=4, total_experts=8, prefill_overlap=True,
|
|
kv_reserve_tokens=0, page_size=16,
|
|
)
|
|
assert (size, pages, overlap) == expected
|
|
|
|
class StubMethod:
|
|
def slot_limit(self):
|
|
return 5
|
|
|
|
size, _, _ = engine._resolve_auto_moe_cache_size(StubConfig(), StubBanks(), StubMethod())
|
|
assert size == 5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# offload-cache sizing guard + auto-resolution (_require_offload_cache_size / _adjust_config),
|
|
# the floor rule compute_cache_floors documents above.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _offload_engine_config(**overrides):
|
|
"""A frozen EngineConfig for a quantized-experts MoE checkpoint in the bare-invocation state
|
|
(moe_strategy="auto") unless overridden — the shared fixture for the _adjust_config tests."""
|
|
from freetoken.distributed import DistributedInfo
|
|
from freetoken.engine.config import EngineConfig
|
|
|
|
config = EngineConfig(
|
|
model_path="/tmp/freetoken-test-model",
|
|
tp_info=DistributedInfo(rank=0, size=1),
|
|
dtype=torch.bfloat16,
|
|
attention_backend="triton" if torch.version.hip is not None else "fi",
|
|
**overrides,
|
|
)
|
|
object.__setattr__(
|
|
config,
|
|
"model_config",
|
|
SimpleNamespace(
|
|
has_swa_attention=False,
|
|
has_linear_attention=False,
|
|
is_moe=True,
|
|
num_layers=10,
|
|
num_moe_layers=10,
|
|
num_experts=8,
|
|
expert_quant="nvfp4",
|
|
moe_strategy="auto",
|
|
),
|
|
)
|
|
return config
|
|
|
|
|
|
def test_guard_passes_when_size_covers_one_expert_per_layer():
|
|
from freetoken.engine.engine import _require_offload_cache_size
|
|
|
|
_require_offload_cache_size(cache_size=128, num_experts=128) # no raise
|
|
|
|
|
|
def test_guard_raises_actionable_error_when_too_small():
|
|
from freetoken.engine.engine import _require_offload_cache_size
|
|
|
|
with pytest.raises(ValueError) as exc:
|
|
_require_offload_cache_size(cache_size=0, num_experts=128)
|
|
msg = str(exc.value)
|
|
assert "128" in msg and "moe-cache" in msg
|
|
|
|
|
|
def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backend(monkeypatch):
|
|
"""Bare `ft serve <FTW MoE checkpoint>`: no --moe-backend, no --moe-cache-* flags at all.
|
|
|
|
args.py's parse-time default only fires when the backend is *already*
|
|
offload-family at parse time -- but a bare invocation leaves moe_strategy="auto" at parse
|
|
time, and the "auto" -> offload/cpu/hybrid resolution only happens later, in _adjust_config,
|
|
once the model_config (and its expert_quant) is known. This proves the engine-level
|
|
resolution: a quantized-experts model auto-resolving to an offload-family backend also gets
|
|
moe_cache_auto=True, so _init_offload_moe_cache's _require_offload_cache_size guard is never
|
|
reached with moe_cache_size still 0.
|
|
"""
|
|
import freetoken.engine.engine as engine_module
|
|
from freetoken.engine.engine import _adjust_config
|
|
from freetoken.moe import is_offload_moe_strategy
|
|
|
|
# a unified-memory host (GB10) resolves auto to fused instead
|
|
monkeypatch.setattr(engine_module, "_is_unified_memory_gpu", lambda index=None: False)
|
|
config = _offload_engine_config()
|
|
_adjust_config(config)
|
|
|
|
# Which member of the family gets picked is not this test's claim. Assert the family, not the member.
|
|
assert is_offload_moe_strategy(config.moe_strategy)
|
|
assert config.moe_cache_auto is True
|
|
assert config.moe_cache_size == 0 # still unresolved -- the scheduler sizes it from VRAM
|
|
|
|
|
|
def test_page_table_width_covers_whole_trailing_pages():
|
|
# _write_page_table writes WHOLE trailing pages, so the width must reach the last
|
|
# page's end, not just the next multiple of 32 (DSV4's P=128 exposed the gap).
|
|
from freetoken.engine.engine import _page_table_width
|
|
|
|
assert _page_table_width(4001, 128) == 4096 # align32 alone gave 4032 -> OOB
|
|
assert _page_table_width(4096, 128) == 4096 # page-aligned length unchanged
|
|
assert _page_table_width(100, 1) == 128 # page_size 1 degenerates to align32
|
|
assert _page_table_width(33, 128) == 128
|
|
for max_seq_len in (1, 31, 33, 4001, 4095, 4096):
|
|
for page_size in (1, 32, 64, 128):
|
|
w = _page_table_width(max_seq_len, page_size)
|
|
last_col = -(-max_seq_len // page_size) * page_size - 1
|
|
assert w > last_col and w % 32 == 0
|
|
|
|
|
|
def _generic_rotary_cfg(max_position, override, scaling=None):
|
|
from types import SimpleNamespace
|
|
|
|
from freetoken.models.config import RotaryConfig
|
|
|
|
model_config = SimpleNamespace(
|
|
single_stream_only=False, is_moe=False, expert_quant="none",
|
|
has_swa_attention=False, has_linear_attention=False,
|
|
rotary_config=RotaryConfig(
|
|
head_dim=128, rotary_dim=128, max_position=max_position, base=1e4, scaling=scaling
|
|
),
|
|
)
|
|
|
|
class Cfg:
|
|
moe_cache_auto = False
|
|
moe_cache_size = 0
|
|
moe_cache_rate = None
|
|
moe_strategy = "auto"
|
|
max_running_req = 4
|
|
cuda_graph_max_bs = 2
|
|
cuda_graph_bs = [1, 2]
|
|
max_seq_len = 1024
|
|
page_size = 1
|
|
attention_backend = "triton"
|
|
num_page_override = None
|
|
num_token_override = None
|
|
max_seq_len_override = None
|
|
|
|
@property
|
|
def model_config(self):
|
|
return model_config
|
|
|
|
cfg = Cfg()
|
|
object.__setattr__(cfg, "max_seq_len_override", override)
|
|
return cfg
|
|
|
|
|
|
def test_adjust_config_rejects_override_past_rope_table():
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
with pytest.raises(ValueError, match="rope table"):
|
|
_adjust_config(_generic_rotary_cfg(max_position=1024, override=2048))
|
|
|
|
|
|
def test_adjust_config_allows_override_at_rope_table_boundary():
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
_adjust_config(_generic_rotary_cfg(max_position=1024, override=1024)) # must not raise
|
|
|
|
|
|
YARN_4X = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 1024}
|
|
|
|
|
|
def test_adjust_config_allows_override_to_the_yarn_extended_table():
|
|
# A YaRN override that leaves max_position_embeddings at the trained length (Qwen3.8's 1M
|
|
# recipe) serves original * factor positions; the gate must follow the table, not the config.
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
_adjust_config(_generic_rotary_cfg(max_position=1024, override=4096, scaling=YARN_4X))
|
|
with pytest.raises(ValueError, match=r"rope table \(4096 positions\)"):
|
|
_adjust_config(_generic_rotary_cfg(max_position=1024, override=4097, scaling=YARN_4X))
|
|
|
|
|
|
def test_adjust_config_rope_gate_exempts_dsv4():
|
|
# DSV4 sizes its own rope table from the resolved max_seq_len (_adjust_dsv4_config),
|
|
# so the generic gate must not fire even when the override dwarfs max_position.
|
|
from types import SimpleNamespace
|
|
|
|
from freetoken.engine.engine import _adjust_config
|
|
|
|
cfg = _dsv4_adjust_cfg(max_seq_len_override=10_000_000)
|
|
cfg.model_config.rotary_config = SimpleNamespace(max_position=1024)
|
|
_adjust_config(cfg) # must not raise
|
|
|
|
|
|
# ---- _pin_budget_bytes: host bytes already pinned outside the expert banks ----
|
|
|
|
|
|
def test_reserved_subtracts_from_the_cap(monkeypatch):
|
|
monkeypatch.setenv("FREETOKEN_PIN_BUDGET_GB", "2")
|
|
assert _pin_budget_bytes() == 2 * 2**30
|
|
assert _pin_budget_bytes(reserved=2**30) == 2**30
|
|
assert _pin_budget_bytes(reserved=4 * 2**30) == 0
|
|
|
|
|
|
def test_uncapped_platform_stays_uncapped(monkeypatch):
|
|
monkeypatch.delenv("FREETOKEN_PIN_BUDGET_GB", raising=False)
|
|
if hasattr(os, "uname") and "microsoft" in os.uname().release.lower():
|
|
pytest.skip("WSL caps pinning")
|
|
assert _pin_budget_bytes(reserved=2**30) is None
|
|
|
|
|
|
def test_moe_slot_pool_split(tmp_path):
|
|
import json
|
|
|
|
from freetoken.moe.pool_policy import group_layers, load_profile, make_pools, split_slots
|
|
|
|
pools = make_pools(group_layers("per-layer", [0] * 4, [1] * 4), num_experts=16, min_num_slots=8, slot_limit=None)
|
|
assert split_slots(41, pools) == [11, 10, 10, 10]
|
|
with pytest.raises(ValueError, match="need"):
|
|
split_slots(31, pools)
|
|
with pytest.raises(ValueError, match="hold"):
|
|
split_slots(65, pools)
|
|
# one pool takes the whole cache, as before pools existed
|
|
assert split_slots(100, make_pools(group_layers("unified", [0] * 4, [1] * 4), 16, 8, None)) == [100]
|
|
# unified pools one bank layout each, widest slots first
|
|
assert group_layers("unified", ["a", "b", "a", "b"], [1, 3, 1, 3]) == [[1, 3], [0, 2]]
|
|
|
|
# layer 0 uses two experts, layer 1 has a hot half and a cold half, layers 2 and 3 use all 16 evenly
|
|
counts = [[5] * 2 + [0] * 14, [3] * 8 + [1] * 8, [1] * 16, [1] * 16]
|
|
path = tmp_path / "profile.json"
|
|
path.write_text(json.dumps({"model": "toy", "expert_counts": counts}))
|
|
assert split_slots(48, pools, load_profile(path, 4, 16)) == [8, 8, 16, 16]
|
|
with pytest.raises(ValueError, match="needs"):
|
|
load_profile(path, 4, 32)
|
|
|
|
|
|
|
|
def test_moe_pool_flags_and_decode_batch_clamp():
|
|
from unittest.mock import patch
|
|
|
|
from freetoken.engine.engine import _clamp_decode_batch_to_pools
|
|
from freetoken.server.args import parse_args
|
|
|
|
hf = SimpleNamespace(to_dict=lambda: {"architectures": ["Qwen3MoeForCausalLM"], "torch_dtype": "bfloat16"})
|
|
with patch("freetoken.utils.cached_load_hf_config", lambda _path: hf):
|
|
args, _ = parse_args([
|
|
"--model", "/models/anon", "--moe-cache-mem", "1.5GiB",
|
|
"--moe-cache-scope", "per-layer", "--moe-cache-split", "profile:/p.json",
|
|
])
|
|
assert (args.moe_cache_mem, args.moe_cache_scope, args.moe_cache_split) == (3 << 29, "per-layer", "profile:/p.json")
|
|
for bad in (["--moe-cache-mem", "12"], ["--moe-cache-mem", "1GiB", "--moe-cache-size", "512"]):
|
|
with pytest.raises(SystemExit):
|
|
parse_args(["--model", "/models/anon", *bad])
|
|
|
|
# GPU decode needs batch * top_k distinct experts to fit the smallest pool below num_experts
|
|
config = SimpleNamespace(
|
|
model_config=SimpleNamespace(num_experts=16, num_experts_per_tok=4),
|
|
max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4], decode_rows_per_req=1,
|
|
)
|
|
_clamp_decode_batch_to_pools(config, [12, 16, 9])
|
|
assert (config.max_running_req, config.cuda_graph_max_bs, config.cuda_graph_bs) == (2, 2, [1, 2])
|
|
# two rows per request route twice the experts: 9 // (4 * 2) leaves one request
|
|
config = SimpleNamespace(
|
|
model_config=SimpleNamespace(num_experts=16, num_experts_per_tok=4),
|
|
max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4], decode_rows_per_req=2,
|
|
)
|
|
_clamp_decode_batch_to_pools(config, [12, 16, 9])
|
|
assert (config.max_running_req, config.cuda_graph_max_bs, config.cuda_graph_bs) == (1, 1, [1])
|