feat(kimi-k3): serve MegaMoE at EP sizes that do not divide the experts

Kimi-K3 opts into static expert placement on its attention-DP MegaMoE
path: with --ep-num-redundant-experts R the routed experts hold 896 + R
slots, the router maps logical ids to this rank's static replica, and
the loader fills every placed slot. EP20 runs with R=4 (900 slots, 45
per rank). Placement is refused on every other K3 MoE path.

The EAGLE3 MLA drafter is one dense layer whose config inherits the
target's expert count; its loader no longer builds a routed-expert plan,
which failed at EP sizes that do not divide 896.

Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Enwei Zhu
2026-11-08 14:30:41 +00:00
co-authored by Claude Opus 5.5
parent a4bd0d5d5c
commit 2cb578c08b
8 changed files with 614 additions and 18 deletions
+2 -1
View File
@@ -568,7 +568,8 @@ compatible fallback instead of importing CUDA IPC handles across hosts.
| `--eplb-rebalance-layers-per-chunk` | MoE layers whose experts move in one scheduling round; required with `--enable-eplb`, `1..num MoE layers`. Fewer layers per chunk bound the per-round stall. |
All of these apply only to models that opt in to expert placement
(LongCat-Flash); other models refuse them at startup. See
(LongCat-Flash; Kimi-K3 on its attention-DP MegaMoE path with a `static`
dispatch and no load recording); other models refuse them at startup. See
[static expert placement](../serving/parallelism.md#static-expert-placement-with-redundant-experts)
for the record → place → route flow and the two dispatch flavors, and
[dynamic expert rebalancing](../serving/parallelism.md#dynamic-expert-rebalancing)
+9 -2
View File
@@ -261,6 +261,9 @@ Select the transport with `--all2all-backend`:
For NVIDIA NVFP4 checkpoints, `--moe-backend mega_moe` replaces routed
dispatch, SiTU expert computation, and combine with MegaMoE. It requires
`--all2all-backend none`, keeps routing weights after FC2, and returns BF16 outputs.
An EP size that does not divide the 896 routed experts takes redundant slots
([static expert placement](#static-expert-placement-with-redundant-experts)),
for example `--ep-num-redundant-experts 4 --ep-dispatch-algorithm static` at EP20.
The AG/RS and FlashInfer transports quantize NVFP4 activations before dispatch and transfer their
block scales alongside the routing IDs and weights. Combine outputs remain BF16.
@@ -456,11 +459,15 @@ recorded load with DeepSeek's EPLB algorithm — once at startup from a
profile record, or online from the live counters ([dynamic
rebalancing](#dynamic-expert-rebalancing) below). Models opt in explicitly
(`supports_expert_placement` on the model class) by building their MoE layers
from the placement and loading every placed slot — LongCat-Flash does.
from the placement and loading every placed slot — LongCat-Flash does, and
so does Kimi-K3 on its attention-DP `--moe-backend mega_moe` path (static
dispatch, no load recording or online rebalancing).
TokenSpeed refuses the placement flags for any other model, so it
never installs a placement only to ignore it.
`--ep-num-redundant-experts` also needs expert parallelism (`ep_size > 1`):
replicas live on other ranks.
replicas live on other ranks. It also serves an EP size that does not divide
the routed experts: Kimi-K3's 896 experts run at EP20 with
`--ep-num-redundant-experts 4` (900 slots, 45 per rank).
1. **Record** the load of a representative workload. Serve with
`--expert-distribution-recorder-mode stat --ep-dispatch-algorithm
@@ -2095,6 +2095,8 @@ class DeepseekV3Model(nn.Module):
class DeepseekV3ForCausalLM(BaseCausalLM):
model_cls = DeepseekV3Model
# Whether the checkpoint carries routed experts for this model's MoE layers.
has_routed_experts = True
def __init__(
self,
@@ -2219,10 +2221,14 @@ class DeepseekV3ForCausalLM(BaseCausalLM):
# by the checkpoint loader.
moe_loader = build_moe_checkpoint_loader(
params_dict=moe_params_dict,
expert_schema=ExpertCheckpointSchema(
gate_proj_name="gate_proj",
down_proj_name="down_proj",
up_proj_name="up_proj",
expert_schema=(
ExpertCheckpointSchema(
gate_proj_name="gate_proj",
down_proj_name="down_proj",
up_proj_name="up_proj",
)
if self.has_routed_experts
else None
),
num_experts=self.config.n_routed_experts,
ep_rank=self.mapping.moe.ep_rank,
@@ -2718,6 +2724,9 @@ class Eagle3DeepseekV2ForCausalLM(DeepseekV3ForCausalLM):
concatenated [embeds || hidden_states] as input.
"""
# One dense decoder layer; the config's MoE fields describe the target model.
has_routed_experts = False
def __init__(
self,
config: PretrainedConfig,
+144 -4
View File
@@ -58,7 +58,7 @@ from __future__ import annotations
import logging
import re
from collections.abc import Iterable
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, ClassVar
import torch
import torch.nn.functional as F
@@ -145,6 +145,7 @@ from tokenspeed.runtime.layers.moe.loader import build_moe_checkpoint_loader
from tokenspeed.runtime.layers.moe.peer_weights import MoEPeerWeights
from tokenspeed.runtime.layers.moe.schema import ExpertCheckpointSchema
from tokenspeed.runtime.layers.moe.topk import (
ExpertLocationDispatchInfo,
StandardTopKOutput,
TopK,
TopKOutput,
@@ -196,6 +197,11 @@ from tokenspeed.runtime.models.kimi_k3_projection import (
validate_projection_settings,
)
from tokenspeed.runtime.models.moonvit import MoonViTVisionPath
from tokenspeed.runtime.moe.expert_location import (
ExpertLocationMetadata,
ModelConfigForExpertLocation,
get_global_expert_location_metadata,
)
from tokenspeed.runtime.multimodal.embedder import (
EncoderSpec,
VisionEmbedder,
@@ -1883,6 +1889,66 @@ def _amd_moe_join_lane(like: torch.Tensor, width: int) -> torch.Tensor | None:
return lane
def _check_k3_expert_placement(
placement: ExpertLocationMetadata,
config: KimiLinearConfig,
mapping: Mapping,
layer_index: int,
execution_plan: Kimi3MoEExecutionPlan,
peer_weights: MoEPeerWeights | None,
) -> None:
"""Refuse a placement this K3 MoE layer cannot route through.
Only the attention-DP MegaMoE path consumes one: every rank routes its own
tokens, the router maps them to this rank's static replica choice, and
MegaMoE dispatches each route to the rank owning that physical slot.
"""
if placement.num_logical_experts != config.num_experts:
raise ValueError(
f"expert placement has {placement.num_logical_experts} logical experts, "
f"Kimi-K3 routes {config.num_experts}"
)
if not 0 <= layer_index < placement.num_layers:
raise ValueError(
f"Kimi-K3 MoE layer {layer_index} is outside the placement's "
f"{placement.num_layers} layers"
)
if placement.ep_size != mapping.moe.ep_size:
raise ValueError(
f"expert placement spans ep_size={placement.ep_size}, the MoE mapping "
f"has ep_size={mapping.moe.ep_size}"
)
if not (
execution_plan.use_mega_moe
and mapping.attn.dp_size > 1
and mapping.moe.tp_size == 1
and mapping.moe.ep_size > 1
):
raise ValueError(
"Kimi-K3 expert placement requires attention DP with --moe-backend "
"mega_moe and MoE EP only (moe tp_size 1)"
)
if peer_weights is not None:
raise ValueError(
"Kimi-K3 expert placement does not support --moe-weight-sharing-size"
)
if placement.physical_load is not None:
raise ValueError(
"Kimi-K3 expert placement does not support "
"--expert-distribution-recorder-mode or --enable-eplb: the "
"attention-DP MoE routes local rows only, which the load row mask "
"does not lay out, and online rebalancing needs "
"routed_experts_weights_of_layer"
)
algorithm = global_server_args_dict["ep_dispatch_algorithm"]
if algorithm != "static":
raise ValueError(
"Kimi-K3 expert placement needs --ep-dispatch-algorithm static: the "
"router writes MegaMoE's id buffer through this rank's static "
f"replica map, not {algorithm}"
)
class KimiLinearMoE(nn.Module):
"""Kimi-K3 MoE block: sigmoid / noaux_tc router + Latent MoE + shared experts.
@@ -1953,6 +2019,20 @@ class KimiLinearMoE(nn.Module):
f"--all2all-backend {fused_all2all_backend.value}; "
"the fused kernel owns dispatch/combine."
)
# The target's expert placement (None for drafts and plain serving):
# the routed experts hold P = E + R slots and the router emits slot ids.
self.expert_placement: ExpertLocationMetadata | None = (
get_global_expert_location_metadata()
)
if self.expert_placement is not None:
_check_k3_expert_placement(
self.expert_placement,
config,
mapping,
layer_index,
self.execution_plan,
peer_weights,
)
# Router (gate+topk) and shared experts run on this stream during
# graph capture, overlapped with the main-stream routed chain
# (down_proj -> fused SiTU MoE -> up_proj). Collectives stay on the default
@@ -1991,7 +2071,11 @@ class KimiLinearMoE(nn.Module):
# TopK fallback (for example NVFP4, AMD-native, or Marlin SiTU).
self.experts = MoELayer(
top_k=self.top_k,
num_experts=self.num_experts,
num_experts=(
self.num_experts
if self.expert_placement is None
else self.expert_placement.num_physical_experts
),
hidden_size=self.routed_hidden,
intermediate_size=config.moe_intermediate_size,
quant_config=quant_config,
@@ -2054,6 +2138,7 @@ class KimiLinearMoE(nn.Module):
# MXFP4 SiTU and precomputed-TopK NVFP4 SiTU implementations).
self.topk = TopK(
top_k=self.top_k,
layer_id=layer_index,
renormalize=config.moe_renormalize,
use_grouped_topk=config.use_grouped_topk,
num_expert_group=config.num_expert_group,
@@ -2074,6 +2159,16 @@ class KimiLinearMoE(nn.Module):
else torch.float32
),
)
self.expert_dispatch_info: ExpertLocationDispatchInfo | None = None
if self.expert_placement is not None:
self.expert_dispatch_info = ExpertLocationDispatchInfo.init_new(
layer_id=layer_index,
ep_dispatch_algorithm=global_server_args_dict["ep_dispatch_algorithm"],
expert_location_metadata=self.expert_placement,
# MegaMoE fuses its dispatch, so its plan names no all-to-all
# backend, yet each rank routes only its own tokens.
all_to_all_ep=True,
)
# AMD replicates both: no folded AR→GEMM→AR, and its native tail packs the weight.
self._shard_latent_projections = _shard_k3_latent_projection(
@@ -2659,7 +2754,7 @@ class KimiLinearMoE(nn.Module):
router_logits,
output_format=TopKOutputFormat.STANDARD,
num_token_non_padded=None,
expert_location_dispatch_info=None,
expert_location_dispatch_info=self.expert_dispatch_info,
out=routing_out,
)
if fused_inputs is None:
@@ -2969,6 +3064,14 @@ def create_kimi_linear_moe(
implementation = KimiLinearMoEDeepEP
if peer_weights is not None and implementation is not KimiLinearMoE:
raise ValueError("Peer expert weights require the local MoE implementation")
if (
get_global_expert_location_metadata() is not None
and implementation is not KimiLinearMoE
):
raise ValueError(
"Kimi-K3 expert placement requires the attention-DP MegaMoE path, "
"not --all2all-backend deepep"
)
peer_kwargs = {"peer_weights": peer_weights} if peer_weights is not None else {}
return implementation(
**peer_kwargs,
@@ -3986,6 +4089,18 @@ class KimiLinearForCausalLM(BaseCausalLM):
model_cls = KimiLinearModel
@property
def expert_placement(self) -> ExpertLocationMetadata | None:
"""The placement the MoE layers were built with; None routes trivially."""
for layer in self.model.layers:
if (
isinstance(layer, KimiLinearDecoderLayer)
and layer.is_moe_layer
and isinstance(layer.block_sparse_moe, KimiLinearMoE)
):
return layer.block_sparse_moe.expert_placement
return None
def post_quant_warmup(self) -> None:
if self.model.peer_weights is not None:
self.model.peer_weights.finalize()
@@ -4262,12 +4377,19 @@ class KimiLinearForCausalLM(BaseCausalLM):
scale_param.weight_loader(scale_param, fused_s)
return True
# Every placed slot is filled from the logical expert it holds,
# replicas included.
expert_placement = self.expert_placement
moe_loader = build_moe_checkpoint_loader(
params_dict=params_dict,
expert_schema=ExpertCheckpointSchema(
gate_proj_name="w1", up_proj_name="w3", down_proj_name="w2"
),
num_experts=config.num_experts,
num_experts=(
config.num_experts
if expert_placement is None
else expert_placement.num_physical_experts
),
ep_rank=(
self.mapping.moe_weight_group.index(self.mapping.rank)
if self.model.peer_weights is not None
@@ -4278,6 +4400,7 @@ class KimiLinearForCausalLM(BaseCausalLM):
if self.model.peer_weights is not None
else self.mapping.moe.ep_size
),
expert_placement=expert_placement,
)
pp_start = self.model.pp_start_layer
@@ -4471,6 +4594,11 @@ class KimiK3ForConditionalGeneration(nn.Module):
a text ``language_model``. The text path is ``KimiLinearForCausalLM``.
"""
# The attention-DP MegaMoE layers size their slots from the expert
# placement and route through it; ``_check_k3_expert_placement`` refuses
# it on every other MoE path.
supports_expert_placement: ClassVar[bool] = True
def __init__(
self,
config: KimiK3Config,
@@ -4736,5 +4864,17 @@ class KimiK3ForConditionalGeneration(nn.Module):
)
logger.debug(f"Loaded {loaded_vision_weights:d} Kimi-K3 vision tensors.")
@classmethod
def get_model_config_for_expert_location(
cls, config: KimiK3Config
) -> ModelConfigForExpertLocation:
# One placement row per decoder layer: the layer id indexes it.
text_config = config.text_config
return ModelConfigForExpertLocation(
num_layers=text_config.num_hidden_layers,
num_logical_experts=text_config.num_experts,
num_groups=None,
)
EntryClass = [KimiK3ForConditionalGeneration]
@@ -33,8 +33,9 @@ placement can be derived (``--init-expert-location <records>``).
The placement is process-global for the target model
(``set_global_expert_location_metadata``); drafts route their own experts
trivially. Models opt in explicitly (``BaseCausalLM.supports_expert_placement``)
by building their MoE layers from the placement; the placement is refused for
trivially. Models opt in explicitly (``BaseCausalLM.supports_expert_placement``,
or the same class attribute on a model class outside ``BaseCausalLM``) by
building their MoE layers from the placement; the placement is refused for
any other model. Zero experts (LongCat) never enter these tables: the router
keeps them as ``-1`` and maps only real expert ids.
@@ -774,7 +775,9 @@ def compute_logical_to_rank_dispatch_physical_map(
if not 0 <= ep_rank < num_gpus:
raise ValueError(f"ep_rank={ep_rank} is outside the {num_gpus} EP ranks")
num_local = num_physical_experts // num_gpus
nodes = torch.tensor(list(ep_rank_nodes), dtype=torch.int64)
# Host-side table: explicit devices, since model construction may run under a
# CUDA default-device context.
nodes = torch.tensor(list(ep_rank_nodes), dtype=torch.int64, device="cpu")
num_nodes = int(nodes.max().item()) + 1
replicas = logical_to_all_physical_map.cpu().to(torch.int64)
@@ -789,7 +792,10 @@ def compute_logical_to_rank_dispatch_physical_map(
# [layers, experts, gpus] instead of a loop over the gpus.
def first_replica_on(owner: torch.Tensor, count: int) -> torch.Tensor:
first = torch.full(
(num_layers, num_logical_experts, count), -1, dtype=torch.int64
(num_layers, num_logical_experts, count),
-1,
dtype=torch.int64,
device="cpu",
)
for column in reversed(range(max_replicas)):
index = owner[..., column].clamp_min(0).unsqueeze(-1)
@@ -822,6 +828,7 @@ def compute_logical_to_rank_dispatch_physical_map(
column[unassigned] = torch.tensor(
_fair_choices(candidates, k=int(unassigned.sum().item()), r=r),
dtype=torch.int64,
device="cpu",
)
return (
@@ -900,9 +907,14 @@ def build_expert_placement(
from tokenspeed.runtime.models.base.causal_lm import BaseCausalLM
model_class, architecture = get_model_architecture(model_config)
if not (
issubclass(model_class, BaseCausalLM) and model_class.supports_expert_placement
):
# A model class outside BaseCausalLM (Kimi-K3's multimodal wrapper) opts
# in by declaring the flag on itself.
opted_in = (
model_class.supports_expert_placement
if issubclass(model_class, BaseCausalLM)
else vars(model_class).get("supports_expert_placement", False)
)
if opted_in is not True:
raise ValueError(
f"{architecture} does not route through an expert placement; "
"--ep-num-redundant-experts, --init-expert-location, "
@@ -164,6 +164,75 @@ def test_placement_plan_for_896_slots_over_128_ranks_has_7_slots_per_rank():
assert last[0].checkpoint_weight_name == "experts.121.gate_proj."
def test_k3_900_slots_over_20_ranks_hold_every_expert_and_its_replicas():
# Kimi-K3: 896 routed experts do not divide over EP20; four redundant
# slots make 900 (45 per rank), the trivial placement wrapping slots
# 896..899 onto experts 0..3 on the last rank.
schema = ExpertCheckpointSchema(
gate_proj_name="w1", up_proj_name="w3", down_proj_name="w2"
)
physical_to_logical = (torch.arange(900) % 896).repeat(2, 1)
nodes = tuple(r // 4 for r in range(20))
with pytest.raises(ValueError, match="divide evenly"):
build_moe_checkpoint_loader(
params_dict={},
expert_schema=schema,
num_experts=896,
ep_rank=0,
ep_size=20,
)
slots_of_expert: dict[int, list[tuple[int, int]]] = {}
for ep_rank in range(20):
placement = ExpertLocationMetadata.from_physical_to_logical_map(
physical_to_logical, 896, ep_size=20, ep_rank=ep_rank, ep_rank_nodes=nodes
)
assert placement.num_local_physical_experts == 45
plan = _build_placed_expert_plan(
schema, expert_placement=placement, ep_rank=ep_rank
)
layer_plan = [entry for entry in plan if entry.layer_id == 1]
assert len(layer_plan) == 45 * 3
for entry in layer_plan[::3]:
expert = int(entry.checkpoint_weight_name.split(".")[1])
slots_of_expert.setdefault(expert, []).append(
(ep_rank, entry.local_expert_id)
)
assert sorted(slots_of_expert) == list(range(896))
for expert, slots in slots_of_expert.items():
if expert < 4:
assert slots == [(0, expert), (19, 41 + expert)]
else:
assert slots == [(expert // 45, expert % 45)]
# Rank 19 loads expert 2 into its replica slot and expert 860 into slot 5.
placement = ExpertLocationMetadata.from_physical_to_logical_map(
physical_to_logical, 896, ep_size=20, ep_rank=19, ep_rank_nodes=nodes
)
slots = _Slots(45)
loader = build_moe_checkpoint_loader(
params_dict={
f"model.layers.1.block_sparse_moe.experts.{name}": param
for name, param in slots.named_parameters()
},
expert_schema=schema,
num_experts=900,
ep_rank=19,
ep_size=20,
expert_placement=placement,
)
loader.load(
"model.layers.1.block_sparse_moe.experts.2.w2.weight", torch.full((4, 4), 2.0)
)
loader.load(
"model.layers.1.block_sparse_moe.experts.860.w2.weight",
torch.full((4, 4), 860.0),
)
assert slots.writes == [("w2", 43), ("w2", 5)]
assert slots.w2_weight[43, 0, 0] == 2.0 and slots.w2_weight[5, 0, 0] == 860.0
assert not loader.matches("model.layers.1.block_sparse_moe.experts.4.w1.weight")
def test_loader_geometry_must_match_the_placement():
placement = _placement([[0, 1, 2, 3, 0, 2]], 4, 2, ep_rank=0)
with pytest.raises(ValueError, match="physical experts"):
@@ -0,0 +1,356 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Kimi-K3 expert placement: 896 routed experts on 900 slots at EP20."""
import os
import sys
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from transformers import PretrainedConfig
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from ci_system.ci_register import register_cuda_ci # noqa: E402
register_cuda_ci(est_time=30, suite="runtime-1gpu")
from tokenspeed.runtime.configs.kimi_k3_config import ( # noqa: E402
KimiK3Config,
KimiLinearConfig,
)
from tokenspeed.runtime.distributed.mapping import Mapping # noqa: E402
from tokenspeed.runtime.layers.moe.latent import Kimi3MoEExecutionPlan # noqa: E402
from tokenspeed.runtime.layers.moe.topk import ( # noqa: E402
ExpertLocationDispatchInfo,
TopK,
TopKOutputFormat,
)
from tokenspeed.runtime.layers.moe.utils import ( # noqa: E402
All2AllBackend,
MoeBackend,
)
from tokenspeed.runtime.models import kimi_k3 # noqa: E402
from tokenspeed.runtime.models.deepseek_v3 import ( # noqa: E402
Eagle3DeepseekV2ForCausalLM,
)
from tokenspeed.runtime.models.kimi_k3 import ( # noqa: E402
KimiLinearMoE,
_check_k3_expert_placement,
)
from tokenspeed.runtime.moe.expert_location import ( # noqa: E402
ExpertLocationMetadata,
build_expert_placement,
compute_logical_to_rank_dispatch_physical_map,
set_global_expert_location_metadata,
)
_EXPERTS, _SLOTS, _EP, _LAYERS = 896, 900, 20, 4
def _mapping(rank: int) -> Mapping:
return Mapping(
rank=rank,
world_size=_EP,
attn_dp_size=_EP,
moe_ep_size=_EP,
nprocs_per_node=4,
)
def _placement(ep_rank: int, device: str, ep_size: int = _EP):
# Trivial placement, four GPUs per node: experts 0..3 sit on rank 0 (node
# 0) and again in slots 896..899 on rank 19 (node 4).
physical_to_logical = (torch.arange(_SLOTS) % _EXPERTS).repeat(_LAYERS, 1)
return ExpertLocationMetadata.from_physical_to_logical_map(
physical_to_logical.to(device),
_EXPERTS,
ep_size=ep_size,
ep_rank=ep_rank,
ep_rank_nodes=tuple(rank // 4 for rank in range(ep_size)),
)
@pytest.fixture
def static_dispatch(monkeypatch):
monkeypatch.setitem(
kimi_k3.global_server_args_dict, "ep_dispatch_algorithm", "static"
)
def test_k3_entry_opts_in_with_its_text_geometry():
config = KimiK3Config(architectures=["KimiK3ForConditionalGeneration"])
model_config = SimpleNamespace(
hf_config=config, quantization=None, model_profile_architecture=None
)
args = SimpleNamespace(
ep_num_redundant_experts=4,
init_expert_location="trivial",
expert_distribution_recorder_mode=None,
ep_dispatch_algorithm="static",
enable_eplb=False,
device="cpu",
mapping=_mapping(19),
)
placement = build_expert_placement(args, model_config)
assert placement.num_layers == config.text_config.num_hidden_layers
assert placement.num_logical_experts == _EXPERTS
assert placement.num_physical_experts == _SLOTS
assert placement.num_local_physical_experts == 45
assert placement.local_slot_logical_experts(5, 19) == [*range(855, 896), 0, 1, 2, 3]
assert placement.rank_dispatch_map()[5, :4].tolist() == [896, 897, 898, 899]
# Without the redundant slots 896 experts do not divide over EP20.
args.ep_num_redundant_experts = 0
args.expert_distribution_recorder_mode = "stat"
with pytest.raises(ValueError, match="do not divide"):
build_expert_placement(args, model_config)
def test_k3_accepts_a_placement_only_on_the_attention_dp_megamoe_path(
monkeypatch, static_dispatch
):
config = KimiLinearConfig(num_experts=_EXPERTS)
mapping = _mapping(0)
mega_moe = Kimi3MoEExecutionPlan.build(mapping, MoeBackend.MEGA_MOE, None)
_check_k3_expert_placement(_placement(0, "cpu"), config, mapping, 3, mega_moe, None)
trtllm = Kimi3MoEExecutionPlan(
use_native=False,
use_trtllm=True,
use_mega_moe=False,
owns_ep_communication=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
)
attn_tp = Mapping(rank=0, world_size=_EP, attn_tp_size=_EP, moe_ep_size=_EP)
refusals = [
((_placement(0, "cpu"), config, mapping, 3, trtllm), "mega_moe"),
((_placement(0, "cpu"), config, attn_tp, 3, mega_moe), "attention DP"),
(
(
_placement(0, "cpu"),
KimiLinearConfig(num_experts=768),
mapping,
3,
mega_moe,
),
"logical experts",
),
((_placement(0, "cpu"), config, mapping, _LAYERS, mega_moe), "outside"),
((_placement(0, "cpu", ep_size=10), config, mapping, 3, mega_moe), "ep_size"),
]
for args, match in refusals:
with pytest.raises(ValueError, match=match):
_check_k3_expert_placement(*args, None)
recording = _placement(0, "cpu")
recording.enable_load_recording(mapping)
with pytest.raises(ValueError, match="--enable-eplb"):
_check_k3_expert_placement(recording, config, mapping, 3, mega_moe, None)
monkeypatch.setitem(
kimi_k3.global_server_args_dict, "ep_dispatch_algorithm", "dynamic"
)
with pytest.raises(ValueError, match="--ep-dispatch-algorithm static"):
_check_k3_expert_placement(
_placement(0, "cpu"), config, mapping, 3, mega_moe, None
)
def test_k3_moe_builds_physical_slots_and_a_static_router_map(
monkeypatch, static_dispatch
):
class Experts(nn.Module):
"""Stands in for MegaMoE, whose setup needs the 20-rank EP group."""
def __init__(self, **kwargs):
super().__init__()
self.kwargs = kwargs
self.supports_precomputed_topk = True
self.topk_output_format = TopKOutputFormat.STANDARD
self.plan = {"a2a_backend": "none"}
monkeypatch.setattr(kimi_k3, "MoELayer", Experts)
monkeypatch.setattr(kimi_k3, "get_moe_backend", lambda: MoeBackend.MEGA_MOE)
monkeypatch.setattr(kimi_k3, "get_all2all_backend", lambda: All2AllBackend.NONE)
set_global_expert_location_metadata(_placement(19, "cpu"))
try:
layer = KimiLinearMoE(
config=KimiLinearConfig(
hidden_size=64,
routed_expert_hidden_size=32,
moe_intermediate_size=32,
num_experts=_EXPERTS,
num_experts_per_token=16,
num_shared_experts=1,
),
mapping=_mapping(19),
layer_index=3,
model_scope="test",
moe_block_count=1,
quant_config=None,
prefix="moe",
alt_stream=None,
)
finally:
set_global_expert_location_metadata(None)
assert layer.experts.kwargs["num_experts"] == _SLOTS
assert layer.gate.weight.shape[0] == _EXPERTS
assert layer.topk.topk_config.layer_id == 3
info = layer.expert_dispatch_info
assert info.layer_id == 3 and info.replica_dispatch is None
assert info.partial_logical_to_rank_dispatch_physical_map[:5].tolist() == [
896,
897,
898,
899,
4,
]
def test_k3_moe_refuses_a_placement_under_deepep(monkeypatch):
monkeypatch.setattr(kimi_k3, "get_all2all_backend", lambda: All2AllBackend.DEEPEP)
set_global_expert_location_metadata(_placement(0, "cpu"))
try:
with pytest.raises(ValueError, match="deepep"):
kimi_k3.create_kimi_linear_moe(
config=KimiLinearConfig(num_experts=_EXPERTS),
mapping=_mapping(0),
layer_index=3,
model_scope="test",
moe_block_count=1,
quant_config=None,
prefix="moe",
alt_stream=None,
)
finally:
set_global_expert_location_metadata(None)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="K3 routing runs on CUDA")
@pytest.mark.parametrize("tokens", [3, 300]) # fused mapped kernel; gather after
@pytest.mark.parametrize("ep_rank", [0, 5, 16, 19])
def test_k3_router_writes_physical_slot_ids_into_the_megamoe_buffers(
tokens: int, ep_rank: int
):
placement = _placement(ep_rank, "cuda")
info = ExpertLocationDispatchInfo.init_new(
layer_id=3,
ep_dispatch_algorithm="static",
expert_location_metadata=placement,
all_to_all_ep=True,
)
# Experts 0..3, the replicated ones, win every token.
bias = torch.zeros(_EXPERTS, device="cuda")
bias[:4] = 10.0
topk = TopK(
top_k=16,
layer_id=3,
renormalize=True,
use_grouped_topk=True,
num_expert_group=1,
topk_group=1,
correction_bias=bias,
routed_scaling_factor=2.5,
output_format=TopKOutputFormat.STANDARD,
topk_weights_dtype=torch.bfloat16,
)
generator = torch.Generator(device="cuda").manual_seed(0)
logits = torch.randn(tokens, _EXPERTS, device="cuda", generator=generator)
hidden = torch.empty(tokens, 8, device="cuda")
def megamoe_buffers():
# MegaMoE's workspace views: FP32 weights, INT32 ids.
return (
torch.empty(tokens, 16, dtype=torch.float32, device="cuda"),
torch.full((tokens, 16), -1, dtype=torch.int32, device="cuda"),
)
logical = megamoe_buffers()
topk(hidden, logits, expert_location_dispatch_info=None, out=logical)
physical = megamoe_buffers()
routed = topk(hidden, logits, expert_location_dispatch_info=info, out=physical)
assert routed.topk_ids.data_ptr() == physical[1].data_ptr()
assert routed.topk_weights.data_ptr() == physical[0].data_ptr()
assert torch.equal(physical[0], logical[0])
assert torch.equal(physical[1], placement.rank_dispatch_map()[3][logical[1].long()])
# Every slot holds the expert the router chose, on the rank that owns it.
assert torch.equal(
placement.physical_to_logical_map[3][physical[1].long()].to(torch.int32),
logical[1],
)
replicas = physical[1][logical[1] < 4] - logical[1][logical[1] < 4]
if ep_rank == 0:
assert (replicas == 0).all() # its own slots 0..3
elif ep_rank in (16, 19):
assert (replicas == _EXPERTS).all() # rank 19's slots 896..899
else:
assert ((replicas == 0) | (replicas == _EXPERTS)).all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device")
def test_static_rank_dispatch_map_builds_under_a_cuda_default_device():
# Model construction runs with CUDA as the default device; the host-side
# nearest-replica table must not pick it up.
replicas = torch.tensor([[[0, -1], [1, 4], [2, -1], [3, 5]]], dtype=torch.int64)
with torch.device("cuda"):
dispatch = compute_logical_to_rank_dispatch_physical_map(
logical_to_all_physical_map=replicas,
num_physical_experts=6,
ep_rank_nodes=[0, 1],
ep_rank=1,
)
assert dispatch.device.type == "cpu"
# Rank 1 owns slots 3..5: it keeps its own replicas of experts 1 and 3 and
# sends experts 0 and 2 to their only slots.
assert dispatch.tolist() == [[0, 4, 2, 3]]
class _DenseDrafterWeights(Eagle3DeepseekV2ForCausalLM):
"""The EAGLE3 MLA drafter's real weight path over one dense projection;
attention post-processing needs MLA modules this test does not build."""
def __init__(self, config, mapping):
nn.Module.__init__(self)
self.config = config
self.mapping = mapping
self.quant_config = None
self.model = nn.Module()
self.model.midlayer = nn.Module()
self.model.midlayer.mlp = nn.Module()
self.model.midlayer.mlp.down_proj = nn.Linear(4, 2, bias=False)
def post_load_weights(self):
pass
def test_eagle3_mla_drafter_loads_at_an_ep_size_that_does_not_divide_the_experts():
# The drafter's config carries the target's 896 routed experts, but its one
# decoder layer is dense: EP20 must not need an expert partition for it.
drafter = _DenseDrafterWeights(
PretrainedConfig(n_routed_experts=_EXPERTS, num_hidden_layers=1),
_mapping(rank=3),
)
weight = torch.arange(8, dtype=torch.float32).reshape(2, 4)
loaded = drafter.load_weights([("layers.0.mlp.down_proj.weight", weight)])
assert loaded == {"model.midlayer.mlp.down_proj.weight"}
assert torch.equal(drafter.model.midlayer.mlp.down_proj.weight.data, weight)
+2
View File
@@ -415,6 +415,7 @@ def test_attn_dp_exchanges_latents_and_returns_reduced_local_rows(
routed_hidden=width,
top_k=2,
topk=topk,
expert_dispatch_info=None,
experts=SimpleNamespace(
plan={"weight_dtype": "nvfp4" if nvfp4 else "unquant"},
w13_input_scale_quant=quant_scale,
@@ -632,6 +633,7 @@ def test_attn_dp_fused_moe_keeps_inputs_local_and_owns_combine(
mapping=SimpleNamespace(world_size=2, attn=SimpleNamespace(dp_rank=0)),
stream_fork=SimpleNamespace(scope=scope),
topk=mock.Mock(return_value=StandardTopKOutput(weights, ids, None)),
expert_dispatch_info=None,
gate=mock.Mock(return_value=torch.zeros(rows, 2)),
routed_expert_down_proj=mock.Mock(return_value=(latent, None)),
experts=SimpleNamespace(