mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
feat(kimi-k3): serve MegaMoE at EP sizes that do not divide the experts
Kimi-K3 opts into static expert placement on its attention-DP MegaMoE path: with --ep-num-redundant-experts R the routed experts hold 896 + R slots, the router maps logical ids to this rank's static replica, and the loader fills every placed slot. EP20 runs with R=4 (900 slots, 45 per rank). Placement is refused on every other K3 MoE path. The EAGLE3 MLA drafter is one dense layer whose config inherits the target's expert count; its loader no longer builds a routed-expert plan, which failed at EP sizes that do not divide 896. Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a4bd0d5d5c
commit
2cb578c08b
@@ -568,7 +568,8 @@ compatible fallback instead of importing CUDA IPC handles across hosts.
|
||||
| `--eplb-rebalance-layers-per-chunk` | MoE layers whose experts move in one scheduling round; required with `--enable-eplb`, `1..num MoE layers`. Fewer layers per chunk bound the per-round stall. |
|
||||
|
||||
All of these apply only to models that opt in to expert placement
|
||||
(LongCat-Flash); other models refuse them at startup. See
|
||||
(LongCat-Flash; Kimi-K3 on its attention-DP MegaMoE path with a `static`
|
||||
dispatch and no load recording); other models refuse them at startup. See
|
||||
[static expert placement](../serving/parallelism.md#static-expert-placement-with-redundant-experts)
|
||||
for the record → place → route flow and the two dispatch flavors, and
|
||||
[dynamic expert rebalancing](../serving/parallelism.md#dynamic-expert-rebalancing)
|
||||
|
||||
@@ -261,6 +261,9 @@ Select the transport with `--all2all-backend`:
|
||||
For NVIDIA NVFP4 checkpoints, `--moe-backend mega_moe` replaces routed
|
||||
dispatch, SiTU expert computation, and combine with MegaMoE. It requires
|
||||
`--all2all-backend none`, keeps routing weights after FC2, and returns BF16 outputs.
|
||||
An EP size that does not divide the 896 routed experts takes redundant slots
|
||||
([static expert placement](#static-expert-placement-with-redundant-experts)),
|
||||
for example `--ep-num-redundant-experts 4 --ep-dispatch-algorithm static` at EP20.
|
||||
|
||||
The AG/RS and FlashInfer transports quantize NVFP4 activations before dispatch and transfer their
|
||||
block scales alongside the routing IDs and weights. Combine outputs remain BF16.
|
||||
@@ -456,11 +459,15 @@ recorded load with DeepSeek's EPLB algorithm — once at startup from a
|
||||
profile record, or online from the live counters ([dynamic
|
||||
rebalancing](#dynamic-expert-rebalancing) below). Models opt in explicitly
|
||||
(`supports_expert_placement` on the model class) by building their MoE layers
|
||||
from the placement and loading every placed slot — LongCat-Flash does.
|
||||
from the placement and loading every placed slot — LongCat-Flash does, and
|
||||
so does Kimi-K3 on its attention-DP `--moe-backend mega_moe` path (static
|
||||
dispatch, no load recording or online rebalancing).
|
||||
TokenSpeed refuses the placement flags for any other model, so it
|
||||
never installs a placement only to ignore it.
|
||||
`--ep-num-redundant-experts` also needs expert parallelism (`ep_size > 1`):
|
||||
replicas live on other ranks.
|
||||
replicas live on other ranks. It also serves an EP size that does not divide
|
||||
the routed experts: Kimi-K3's 896 experts run at EP20 with
|
||||
`--ep-num-redundant-experts 4` (900 slots, 45 per rank).
|
||||
|
||||
1. **Record** the load of a representative workload. Serve with
|
||||
`--expert-distribution-recorder-mode stat --ep-dispatch-algorithm
|
||||
|
||||
@@ -2095,6 +2095,8 @@ class DeepseekV3Model(nn.Module):
|
||||
|
||||
class DeepseekV3ForCausalLM(BaseCausalLM):
|
||||
model_cls = DeepseekV3Model
|
||||
# Whether the checkpoint carries routed experts for this model's MoE layers.
|
||||
has_routed_experts = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -2219,10 +2221,14 @@ class DeepseekV3ForCausalLM(BaseCausalLM):
|
||||
# by the checkpoint loader.
|
||||
moe_loader = build_moe_checkpoint_loader(
|
||||
params_dict=moe_params_dict,
|
||||
expert_schema=ExpertCheckpointSchema(
|
||||
gate_proj_name="gate_proj",
|
||||
down_proj_name="down_proj",
|
||||
up_proj_name="up_proj",
|
||||
expert_schema=(
|
||||
ExpertCheckpointSchema(
|
||||
gate_proj_name="gate_proj",
|
||||
down_proj_name="down_proj",
|
||||
up_proj_name="up_proj",
|
||||
)
|
||||
if self.has_routed_experts
|
||||
else None
|
||||
),
|
||||
num_experts=self.config.n_routed_experts,
|
||||
ep_rank=self.mapping.moe.ep_rank,
|
||||
@@ -2718,6 +2724,9 @@ class Eagle3DeepseekV2ForCausalLM(DeepseekV3ForCausalLM):
|
||||
concatenated [embeds || hidden_states] as input.
|
||||
"""
|
||||
|
||||
# One dense decoder layer; the config's MoE fields describe the target model.
|
||||
has_routed_experts = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
|
||||
@@ -58,7 +58,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -145,6 +145,7 @@ from tokenspeed.runtime.layers.moe.loader import build_moe_checkpoint_loader
|
||||
from tokenspeed.runtime.layers.moe.peer_weights import MoEPeerWeights
|
||||
from tokenspeed.runtime.layers.moe.schema import ExpertCheckpointSchema
|
||||
from tokenspeed.runtime.layers.moe.topk import (
|
||||
ExpertLocationDispatchInfo,
|
||||
StandardTopKOutput,
|
||||
TopK,
|
||||
TopKOutput,
|
||||
@@ -196,6 +197,11 @@ from tokenspeed.runtime.models.kimi_k3_projection import (
|
||||
validate_projection_settings,
|
||||
)
|
||||
from tokenspeed.runtime.models.moonvit import MoonViTVisionPath
|
||||
from tokenspeed.runtime.moe.expert_location import (
|
||||
ExpertLocationMetadata,
|
||||
ModelConfigForExpertLocation,
|
||||
get_global_expert_location_metadata,
|
||||
)
|
||||
from tokenspeed.runtime.multimodal.embedder import (
|
||||
EncoderSpec,
|
||||
VisionEmbedder,
|
||||
@@ -1883,6 +1889,66 @@ def _amd_moe_join_lane(like: torch.Tensor, width: int) -> torch.Tensor | None:
|
||||
return lane
|
||||
|
||||
|
||||
def _check_k3_expert_placement(
|
||||
placement: ExpertLocationMetadata,
|
||||
config: KimiLinearConfig,
|
||||
mapping: Mapping,
|
||||
layer_index: int,
|
||||
execution_plan: Kimi3MoEExecutionPlan,
|
||||
peer_weights: MoEPeerWeights | None,
|
||||
) -> None:
|
||||
"""Refuse a placement this K3 MoE layer cannot route through.
|
||||
|
||||
Only the attention-DP MegaMoE path consumes one: every rank routes its own
|
||||
tokens, the router maps them to this rank's static replica choice, and
|
||||
MegaMoE dispatches each route to the rank owning that physical slot.
|
||||
"""
|
||||
if placement.num_logical_experts != config.num_experts:
|
||||
raise ValueError(
|
||||
f"expert placement has {placement.num_logical_experts} logical experts, "
|
||||
f"Kimi-K3 routes {config.num_experts}"
|
||||
)
|
||||
if not 0 <= layer_index < placement.num_layers:
|
||||
raise ValueError(
|
||||
f"Kimi-K3 MoE layer {layer_index} is outside the placement's "
|
||||
f"{placement.num_layers} layers"
|
||||
)
|
||||
if placement.ep_size != mapping.moe.ep_size:
|
||||
raise ValueError(
|
||||
f"expert placement spans ep_size={placement.ep_size}, the MoE mapping "
|
||||
f"has ep_size={mapping.moe.ep_size}"
|
||||
)
|
||||
if not (
|
||||
execution_plan.use_mega_moe
|
||||
and mapping.attn.dp_size > 1
|
||||
and mapping.moe.tp_size == 1
|
||||
and mapping.moe.ep_size > 1
|
||||
):
|
||||
raise ValueError(
|
||||
"Kimi-K3 expert placement requires attention DP with --moe-backend "
|
||||
"mega_moe and MoE EP only (moe tp_size 1)"
|
||||
)
|
||||
if peer_weights is not None:
|
||||
raise ValueError(
|
||||
"Kimi-K3 expert placement does not support --moe-weight-sharing-size"
|
||||
)
|
||||
if placement.physical_load is not None:
|
||||
raise ValueError(
|
||||
"Kimi-K3 expert placement does not support "
|
||||
"--expert-distribution-recorder-mode or --enable-eplb: the "
|
||||
"attention-DP MoE routes local rows only, which the load row mask "
|
||||
"does not lay out, and online rebalancing needs "
|
||||
"routed_experts_weights_of_layer"
|
||||
)
|
||||
algorithm = global_server_args_dict["ep_dispatch_algorithm"]
|
||||
if algorithm != "static":
|
||||
raise ValueError(
|
||||
"Kimi-K3 expert placement needs --ep-dispatch-algorithm static: the "
|
||||
"router writes MegaMoE's id buffer through this rank's static "
|
||||
f"replica map, not {algorithm}"
|
||||
)
|
||||
|
||||
|
||||
class KimiLinearMoE(nn.Module):
|
||||
"""Kimi-K3 MoE block: sigmoid / noaux_tc router + Latent MoE + shared experts.
|
||||
|
||||
@@ -1953,6 +2019,20 @@ class KimiLinearMoE(nn.Module):
|
||||
f"--all2all-backend {fused_all2all_backend.value}; "
|
||||
"the fused kernel owns dispatch/combine."
|
||||
)
|
||||
# The target's expert placement (None for drafts and plain serving):
|
||||
# the routed experts hold P = E + R slots and the router emits slot ids.
|
||||
self.expert_placement: ExpertLocationMetadata | None = (
|
||||
get_global_expert_location_metadata()
|
||||
)
|
||||
if self.expert_placement is not None:
|
||||
_check_k3_expert_placement(
|
||||
self.expert_placement,
|
||||
config,
|
||||
mapping,
|
||||
layer_index,
|
||||
self.execution_plan,
|
||||
peer_weights,
|
||||
)
|
||||
# Router (gate+topk) and shared experts run on this stream during
|
||||
# graph capture, overlapped with the main-stream routed chain
|
||||
# (down_proj -> fused SiTU MoE -> up_proj). Collectives stay on the default
|
||||
@@ -1991,7 +2071,11 @@ class KimiLinearMoE(nn.Module):
|
||||
# TopK fallback (for example NVFP4, AMD-native, or Marlin SiTU).
|
||||
self.experts = MoELayer(
|
||||
top_k=self.top_k,
|
||||
num_experts=self.num_experts,
|
||||
num_experts=(
|
||||
self.num_experts
|
||||
if self.expert_placement is None
|
||||
else self.expert_placement.num_physical_experts
|
||||
),
|
||||
hidden_size=self.routed_hidden,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
quant_config=quant_config,
|
||||
@@ -2054,6 +2138,7 @@ class KimiLinearMoE(nn.Module):
|
||||
# MXFP4 SiTU and precomputed-TopK NVFP4 SiTU implementations).
|
||||
self.topk = TopK(
|
||||
top_k=self.top_k,
|
||||
layer_id=layer_index,
|
||||
renormalize=config.moe_renormalize,
|
||||
use_grouped_topk=config.use_grouped_topk,
|
||||
num_expert_group=config.num_expert_group,
|
||||
@@ -2074,6 +2159,16 @@ class KimiLinearMoE(nn.Module):
|
||||
else torch.float32
|
||||
),
|
||||
)
|
||||
self.expert_dispatch_info: ExpertLocationDispatchInfo | None = None
|
||||
if self.expert_placement is not None:
|
||||
self.expert_dispatch_info = ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=layer_index,
|
||||
ep_dispatch_algorithm=global_server_args_dict["ep_dispatch_algorithm"],
|
||||
expert_location_metadata=self.expert_placement,
|
||||
# MegaMoE fuses its dispatch, so its plan names no all-to-all
|
||||
# backend, yet each rank routes only its own tokens.
|
||||
all_to_all_ep=True,
|
||||
)
|
||||
|
||||
# AMD replicates both: no folded AR→GEMM→AR, and its native tail packs the weight.
|
||||
self._shard_latent_projections = _shard_k3_latent_projection(
|
||||
@@ -2659,7 +2754,7 @@ class KimiLinearMoE(nn.Module):
|
||||
router_logits,
|
||||
output_format=TopKOutputFormat.STANDARD,
|
||||
num_token_non_padded=None,
|
||||
expert_location_dispatch_info=None,
|
||||
expert_location_dispatch_info=self.expert_dispatch_info,
|
||||
out=routing_out,
|
||||
)
|
||||
if fused_inputs is None:
|
||||
@@ -2969,6 +3064,14 @@ def create_kimi_linear_moe(
|
||||
implementation = KimiLinearMoEDeepEP
|
||||
if peer_weights is not None and implementation is not KimiLinearMoE:
|
||||
raise ValueError("Peer expert weights require the local MoE implementation")
|
||||
if (
|
||||
get_global_expert_location_metadata() is not None
|
||||
and implementation is not KimiLinearMoE
|
||||
):
|
||||
raise ValueError(
|
||||
"Kimi-K3 expert placement requires the attention-DP MegaMoE path, "
|
||||
"not --all2all-backend deepep"
|
||||
)
|
||||
peer_kwargs = {"peer_weights": peer_weights} if peer_weights is not None else {}
|
||||
return implementation(
|
||||
**peer_kwargs,
|
||||
@@ -3986,6 +4089,18 @@ class KimiLinearForCausalLM(BaseCausalLM):
|
||||
|
||||
model_cls = KimiLinearModel
|
||||
|
||||
@property
|
||||
def expert_placement(self) -> ExpertLocationMetadata | None:
|
||||
"""The placement the MoE layers were built with; None routes trivially."""
|
||||
for layer in self.model.layers:
|
||||
if (
|
||||
isinstance(layer, KimiLinearDecoderLayer)
|
||||
and layer.is_moe_layer
|
||||
and isinstance(layer.block_sparse_moe, KimiLinearMoE)
|
||||
):
|
||||
return layer.block_sparse_moe.expert_placement
|
||||
return None
|
||||
|
||||
def post_quant_warmup(self) -> None:
|
||||
if self.model.peer_weights is not None:
|
||||
self.model.peer_weights.finalize()
|
||||
@@ -4262,12 +4377,19 @@ class KimiLinearForCausalLM(BaseCausalLM):
|
||||
scale_param.weight_loader(scale_param, fused_s)
|
||||
return True
|
||||
|
||||
# Every placed slot is filled from the logical expert it holds,
|
||||
# replicas included.
|
||||
expert_placement = self.expert_placement
|
||||
moe_loader = build_moe_checkpoint_loader(
|
||||
params_dict=params_dict,
|
||||
expert_schema=ExpertCheckpointSchema(
|
||||
gate_proj_name="w1", up_proj_name="w3", down_proj_name="w2"
|
||||
),
|
||||
num_experts=config.num_experts,
|
||||
num_experts=(
|
||||
config.num_experts
|
||||
if expert_placement is None
|
||||
else expert_placement.num_physical_experts
|
||||
),
|
||||
ep_rank=(
|
||||
self.mapping.moe_weight_group.index(self.mapping.rank)
|
||||
if self.model.peer_weights is not None
|
||||
@@ -4278,6 +4400,7 @@ class KimiLinearForCausalLM(BaseCausalLM):
|
||||
if self.model.peer_weights is not None
|
||||
else self.mapping.moe.ep_size
|
||||
),
|
||||
expert_placement=expert_placement,
|
||||
)
|
||||
|
||||
pp_start = self.model.pp_start_layer
|
||||
@@ -4471,6 +4594,11 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
a text ``language_model``. The text path is ``KimiLinearForCausalLM``.
|
||||
"""
|
||||
|
||||
# The attention-DP MegaMoE layers size their slots from the expert
|
||||
# placement and route through it; ``_check_k3_expert_placement`` refuses
|
||||
# it on every other MoE path.
|
||||
supports_expert_placement: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiK3Config,
|
||||
@@ -4736,5 +4864,17 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
)
|
||||
logger.debug(f"Loaded {loaded_vision_weights:d} Kimi-K3 vision tensors.")
|
||||
|
||||
@classmethod
|
||||
def get_model_config_for_expert_location(
|
||||
cls, config: KimiK3Config
|
||||
) -> ModelConfigForExpertLocation:
|
||||
# One placement row per decoder layer: the layer id indexes it.
|
||||
text_config = config.text_config
|
||||
return ModelConfigForExpertLocation(
|
||||
num_layers=text_config.num_hidden_layers,
|
||||
num_logical_experts=text_config.num_experts,
|
||||
num_groups=None,
|
||||
)
|
||||
|
||||
|
||||
EntryClass = [KimiK3ForConditionalGeneration]
|
||||
|
||||
@@ -33,8 +33,9 @@ placement can be derived (``--init-expert-location <records>``).
|
||||
|
||||
The placement is process-global for the target model
|
||||
(``set_global_expert_location_metadata``); drafts route their own experts
|
||||
trivially. Models opt in explicitly (``BaseCausalLM.supports_expert_placement``)
|
||||
by building their MoE layers from the placement; the placement is refused for
|
||||
trivially. Models opt in explicitly (``BaseCausalLM.supports_expert_placement``,
|
||||
or the same class attribute on a model class outside ``BaseCausalLM``) by
|
||||
building their MoE layers from the placement; the placement is refused for
|
||||
any other model. Zero experts (LongCat) never enter these tables: the router
|
||||
keeps them as ``-1`` and maps only real expert ids.
|
||||
|
||||
@@ -774,7 +775,9 @@ def compute_logical_to_rank_dispatch_physical_map(
|
||||
if not 0 <= ep_rank < num_gpus:
|
||||
raise ValueError(f"ep_rank={ep_rank} is outside the {num_gpus} EP ranks")
|
||||
num_local = num_physical_experts // num_gpus
|
||||
nodes = torch.tensor(list(ep_rank_nodes), dtype=torch.int64)
|
||||
# Host-side table: explicit devices, since model construction may run under a
|
||||
# CUDA default-device context.
|
||||
nodes = torch.tensor(list(ep_rank_nodes), dtype=torch.int64, device="cpu")
|
||||
num_nodes = int(nodes.max().item()) + 1
|
||||
|
||||
replicas = logical_to_all_physical_map.cpu().to(torch.int64)
|
||||
@@ -789,7 +792,10 @@ def compute_logical_to_rank_dispatch_physical_map(
|
||||
# [layers, experts, gpus] instead of a loop over the gpus.
|
||||
def first_replica_on(owner: torch.Tensor, count: int) -> torch.Tensor:
|
||||
first = torch.full(
|
||||
(num_layers, num_logical_experts, count), -1, dtype=torch.int64
|
||||
(num_layers, num_logical_experts, count),
|
||||
-1,
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
for column in reversed(range(max_replicas)):
|
||||
index = owner[..., column].clamp_min(0).unsqueeze(-1)
|
||||
@@ -822,6 +828,7 @@ def compute_logical_to_rank_dispatch_physical_map(
|
||||
column[unassigned] = torch.tensor(
|
||||
_fair_choices(candidates, k=int(unassigned.sum().item()), r=r),
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
return (
|
||||
@@ -900,9 +907,14 @@ def build_expert_placement(
|
||||
from tokenspeed.runtime.models.base.causal_lm import BaseCausalLM
|
||||
|
||||
model_class, architecture = get_model_architecture(model_config)
|
||||
if not (
|
||||
issubclass(model_class, BaseCausalLM) and model_class.supports_expert_placement
|
||||
):
|
||||
# A model class outside BaseCausalLM (Kimi-K3's multimodal wrapper) opts
|
||||
# in by declaring the flag on itself.
|
||||
opted_in = (
|
||||
model_class.supports_expert_placement
|
||||
if issubclass(model_class, BaseCausalLM)
|
||||
else vars(model_class).get("supports_expert_placement", False)
|
||||
)
|
||||
if opted_in is not True:
|
||||
raise ValueError(
|
||||
f"{architecture} does not route through an expert placement; "
|
||||
"--ep-num-redundant-experts, --init-expert-location, "
|
||||
|
||||
@@ -164,6 +164,75 @@ def test_placement_plan_for_896_slots_over_128_ranks_has_7_slots_per_rank():
|
||||
assert last[0].checkpoint_weight_name == "experts.121.gate_proj."
|
||||
|
||||
|
||||
def test_k3_900_slots_over_20_ranks_hold_every_expert_and_its_replicas():
|
||||
# Kimi-K3: 896 routed experts do not divide over EP20; four redundant
|
||||
# slots make 900 (45 per rank), the trivial placement wrapping slots
|
||||
# 896..899 onto experts 0..3 on the last rank.
|
||||
schema = ExpertCheckpointSchema(
|
||||
gate_proj_name="w1", up_proj_name="w3", down_proj_name="w2"
|
||||
)
|
||||
physical_to_logical = (torch.arange(900) % 896).repeat(2, 1)
|
||||
nodes = tuple(r // 4 for r in range(20))
|
||||
with pytest.raises(ValueError, match="divide evenly"):
|
||||
build_moe_checkpoint_loader(
|
||||
params_dict={},
|
||||
expert_schema=schema,
|
||||
num_experts=896,
|
||||
ep_rank=0,
|
||||
ep_size=20,
|
||||
)
|
||||
|
||||
slots_of_expert: dict[int, list[tuple[int, int]]] = {}
|
||||
for ep_rank in range(20):
|
||||
placement = ExpertLocationMetadata.from_physical_to_logical_map(
|
||||
physical_to_logical, 896, ep_size=20, ep_rank=ep_rank, ep_rank_nodes=nodes
|
||||
)
|
||||
assert placement.num_local_physical_experts == 45
|
||||
plan = _build_placed_expert_plan(
|
||||
schema, expert_placement=placement, ep_rank=ep_rank
|
||||
)
|
||||
layer_plan = [entry for entry in plan if entry.layer_id == 1]
|
||||
assert len(layer_plan) == 45 * 3
|
||||
for entry in layer_plan[::3]:
|
||||
expert = int(entry.checkpoint_weight_name.split(".")[1])
|
||||
slots_of_expert.setdefault(expert, []).append(
|
||||
(ep_rank, entry.local_expert_id)
|
||||
)
|
||||
assert sorted(slots_of_expert) == list(range(896))
|
||||
for expert, slots in slots_of_expert.items():
|
||||
if expert < 4:
|
||||
assert slots == [(0, expert), (19, 41 + expert)]
|
||||
else:
|
||||
assert slots == [(expert // 45, expert % 45)]
|
||||
|
||||
# Rank 19 loads expert 2 into its replica slot and expert 860 into slot 5.
|
||||
placement = ExpertLocationMetadata.from_physical_to_logical_map(
|
||||
physical_to_logical, 896, ep_size=20, ep_rank=19, ep_rank_nodes=nodes
|
||||
)
|
||||
slots = _Slots(45)
|
||||
loader = build_moe_checkpoint_loader(
|
||||
params_dict={
|
||||
f"model.layers.1.block_sparse_moe.experts.{name}": param
|
||||
for name, param in slots.named_parameters()
|
||||
},
|
||||
expert_schema=schema,
|
||||
num_experts=900,
|
||||
ep_rank=19,
|
||||
ep_size=20,
|
||||
expert_placement=placement,
|
||||
)
|
||||
loader.load(
|
||||
"model.layers.1.block_sparse_moe.experts.2.w2.weight", torch.full((4, 4), 2.0)
|
||||
)
|
||||
loader.load(
|
||||
"model.layers.1.block_sparse_moe.experts.860.w2.weight",
|
||||
torch.full((4, 4), 860.0),
|
||||
)
|
||||
assert slots.writes == [("w2", 43), ("w2", 5)]
|
||||
assert slots.w2_weight[43, 0, 0] == 2.0 and slots.w2_weight[5, 0, 0] == 860.0
|
||||
assert not loader.matches("model.layers.1.block_sparse_moe.experts.4.w1.weight")
|
||||
|
||||
|
||||
def test_loader_geometry_must_match_the_placement():
|
||||
placement = _placement([[0, 1, 2, 3, 0, 2]], 4, 2, ep_rank=0)
|
||||
with pytest.raises(ValueError, match="physical experts"):
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Kimi-K3 expert placement: 896 routed experts on 900 slots at EP20."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from ci_system.ci_register import register_cuda_ci # noqa: E402
|
||||
|
||||
register_cuda_ci(est_time=30, suite="runtime-1gpu")
|
||||
|
||||
from tokenspeed.runtime.configs.kimi_k3_config import ( # noqa: E402
|
||||
KimiK3Config,
|
||||
KimiLinearConfig,
|
||||
)
|
||||
from tokenspeed.runtime.distributed.mapping import Mapping # noqa: E402
|
||||
from tokenspeed.runtime.layers.moe.latent import Kimi3MoEExecutionPlan # noqa: E402
|
||||
from tokenspeed.runtime.layers.moe.topk import ( # noqa: E402
|
||||
ExpertLocationDispatchInfo,
|
||||
TopK,
|
||||
TopKOutputFormat,
|
||||
)
|
||||
from tokenspeed.runtime.layers.moe.utils import ( # noqa: E402
|
||||
All2AllBackend,
|
||||
MoeBackend,
|
||||
)
|
||||
from tokenspeed.runtime.models import kimi_k3 # noqa: E402
|
||||
from tokenspeed.runtime.models.deepseek_v3 import ( # noqa: E402
|
||||
Eagle3DeepseekV2ForCausalLM,
|
||||
)
|
||||
from tokenspeed.runtime.models.kimi_k3 import ( # noqa: E402
|
||||
KimiLinearMoE,
|
||||
_check_k3_expert_placement,
|
||||
)
|
||||
from tokenspeed.runtime.moe.expert_location import ( # noqa: E402
|
||||
ExpertLocationMetadata,
|
||||
build_expert_placement,
|
||||
compute_logical_to_rank_dispatch_physical_map,
|
||||
set_global_expert_location_metadata,
|
||||
)
|
||||
|
||||
_EXPERTS, _SLOTS, _EP, _LAYERS = 896, 900, 20, 4
|
||||
|
||||
|
||||
def _mapping(rank: int) -> Mapping:
|
||||
return Mapping(
|
||||
rank=rank,
|
||||
world_size=_EP,
|
||||
attn_dp_size=_EP,
|
||||
moe_ep_size=_EP,
|
||||
nprocs_per_node=4,
|
||||
)
|
||||
|
||||
|
||||
def _placement(ep_rank: int, device: str, ep_size: int = _EP):
|
||||
# Trivial placement, four GPUs per node: experts 0..3 sit on rank 0 (node
|
||||
# 0) and again in slots 896..899 on rank 19 (node 4).
|
||||
physical_to_logical = (torch.arange(_SLOTS) % _EXPERTS).repeat(_LAYERS, 1)
|
||||
return ExpertLocationMetadata.from_physical_to_logical_map(
|
||||
physical_to_logical.to(device),
|
||||
_EXPERTS,
|
||||
ep_size=ep_size,
|
||||
ep_rank=ep_rank,
|
||||
ep_rank_nodes=tuple(rank // 4 for rank in range(ep_size)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def static_dispatch(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
kimi_k3.global_server_args_dict, "ep_dispatch_algorithm", "static"
|
||||
)
|
||||
|
||||
|
||||
def test_k3_entry_opts_in_with_its_text_geometry():
|
||||
config = KimiK3Config(architectures=["KimiK3ForConditionalGeneration"])
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=config, quantization=None, model_profile_architecture=None
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
ep_num_redundant_experts=4,
|
||||
init_expert_location="trivial",
|
||||
expert_distribution_recorder_mode=None,
|
||||
ep_dispatch_algorithm="static",
|
||||
enable_eplb=False,
|
||||
device="cpu",
|
||||
mapping=_mapping(19),
|
||||
)
|
||||
placement = build_expert_placement(args, model_config)
|
||||
assert placement.num_layers == config.text_config.num_hidden_layers
|
||||
assert placement.num_logical_experts == _EXPERTS
|
||||
assert placement.num_physical_experts == _SLOTS
|
||||
assert placement.num_local_physical_experts == 45
|
||||
assert placement.local_slot_logical_experts(5, 19) == [*range(855, 896), 0, 1, 2, 3]
|
||||
assert placement.rank_dispatch_map()[5, :4].tolist() == [896, 897, 898, 899]
|
||||
# Without the redundant slots 896 experts do not divide over EP20.
|
||||
args.ep_num_redundant_experts = 0
|
||||
args.expert_distribution_recorder_mode = "stat"
|
||||
with pytest.raises(ValueError, match="do not divide"):
|
||||
build_expert_placement(args, model_config)
|
||||
|
||||
|
||||
def test_k3_accepts_a_placement_only_on_the_attention_dp_megamoe_path(
|
||||
monkeypatch, static_dispatch
|
||||
):
|
||||
config = KimiLinearConfig(num_experts=_EXPERTS)
|
||||
mapping = _mapping(0)
|
||||
mega_moe = Kimi3MoEExecutionPlan.build(mapping, MoeBackend.MEGA_MOE, None)
|
||||
_check_k3_expert_placement(_placement(0, "cpu"), config, mapping, 3, mega_moe, None)
|
||||
|
||||
trtllm = Kimi3MoEExecutionPlan(
|
||||
use_native=False,
|
||||
use_trtllm=True,
|
||||
use_mega_moe=False,
|
||||
owns_ep_communication=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
)
|
||||
attn_tp = Mapping(rank=0, world_size=_EP, attn_tp_size=_EP, moe_ep_size=_EP)
|
||||
refusals = [
|
||||
((_placement(0, "cpu"), config, mapping, 3, trtllm), "mega_moe"),
|
||||
((_placement(0, "cpu"), config, attn_tp, 3, mega_moe), "attention DP"),
|
||||
(
|
||||
(
|
||||
_placement(0, "cpu"),
|
||||
KimiLinearConfig(num_experts=768),
|
||||
mapping,
|
||||
3,
|
||||
mega_moe,
|
||||
),
|
||||
"logical experts",
|
||||
),
|
||||
((_placement(0, "cpu"), config, mapping, _LAYERS, mega_moe), "outside"),
|
||||
((_placement(0, "cpu", ep_size=10), config, mapping, 3, mega_moe), "ep_size"),
|
||||
]
|
||||
for args, match in refusals:
|
||||
with pytest.raises(ValueError, match=match):
|
||||
_check_k3_expert_placement(*args, None)
|
||||
|
||||
recording = _placement(0, "cpu")
|
||||
recording.enable_load_recording(mapping)
|
||||
with pytest.raises(ValueError, match="--enable-eplb"):
|
||||
_check_k3_expert_placement(recording, config, mapping, 3, mega_moe, None)
|
||||
monkeypatch.setitem(
|
||||
kimi_k3.global_server_args_dict, "ep_dispatch_algorithm", "dynamic"
|
||||
)
|
||||
with pytest.raises(ValueError, match="--ep-dispatch-algorithm static"):
|
||||
_check_k3_expert_placement(
|
||||
_placement(0, "cpu"), config, mapping, 3, mega_moe, None
|
||||
)
|
||||
|
||||
|
||||
def test_k3_moe_builds_physical_slots_and_a_static_router_map(
|
||||
monkeypatch, static_dispatch
|
||||
):
|
||||
class Experts(nn.Module):
|
||||
"""Stands in for MegaMoE, whose setup needs the 20-rank EP group."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.kwargs = kwargs
|
||||
self.supports_precomputed_topk = True
|
||||
self.topk_output_format = TopKOutputFormat.STANDARD
|
||||
self.plan = {"a2a_backend": "none"}
|
||||
|
||||
monkeypatch.setattr(kimi_k3, "MoELayer", Experts)
|
||||
monkeypatch.setattr(kimi_k3, "get_moe_backend", lambda: MoeBackend.MEGA_MOE)
|
||||
monkeypatch.setattr(kimi_k3, "get_all2all_backend", lambda: All2AllBackend.NONE)
|
||||
set_global_expert_location_metadata(_placement(19, "cpu"))
|
||||
try:
|
||||
layer = KimiLinearMoE(
|
||||
config=KimiLinearConfig(
|
||||
hidden_size=64,
|
||||
routed_expert_hidden_size=32,
|
||||
moe_intermediate_size=32,
|
||||
num_experts=_EXPERTS,
|
||||
num_experts_per_token=16,
|
||||
num_shared_experts=1,
|
||||
),
|
||||
mapping=_mapping(19),
|
||||
layer_index=3,
|
||||
model_scope="test",
|
||||
moe_block_count=1,
|
||||
quant_config=None,
|
||||
prefix="moe",
|
||||
alt_stream=None,
|
||||
)
|
||||
finally:
|
||||
set_global_expert_location_metadata(None)
|
||||
assert layer.experts.kwargs["num_experts"] == _SLOTS
|
||||
assert layer.gate.weight.shape[0] == _EXPERTS
|
||||
assert layer.topk.topk_config.layer_id == 3
|
||||
info = layer.expert_dispatch_info
|
||||
assert info.layer_id == 3 and info.replica_dispatch is None
|
||||
assert info.partial_logical_to_rank_dispatch_physical_map[:5].tolist() == [
|
||||
896,
|
||||
897,
|
||||
898,
|
||||
899,
|
||||
4,
|
||||
]
|
||||
|
||||
|
||||
def test_k3_moe_refuses_a_placement_under_deepep(monkeypatch):
|
||||
monkeypatch.setattr(kimi_k3, "get_all2all_backend", lambda: All2AllBackend.DEEPEP)
|
||||
set_global_expert_location_metadata(_placement(0, "cpu"))
|
||||
try:
|
||||
with pytest.raises(ValueError, match="deepep"):
|
||||
kimi_k3.create_kimi_linear_moe(
|
||||
config=KimiLinearConfig(num_experts=_EXPERTS),
|
||||
mapping=_mapping(0),
|
||||
layer_index=3,
|
||||
model_scope="test",
|
||||
moe_block_count=1,
|
||||
quant_config=None,
|
||||
prefix="moe",
|
||||
alt_stream=None,
|
||||
)
|
||||
finally:
|
||||
set_global_expert_location_metadata(None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="K3 routing runs on CUDA")
|
||||
@pytest.mark.parametrize("tokens", [3, 300]) # fused mapped kernel; gather after
|
||||
@pytest.mark.parametrize("ep_rank", [0, 5, 16, 19])
|
||||
def test_k3_router_writes_physical_slot_ids_into_the_megamoe_buffers(
|
||||
tokens: int, ep_rank: int
|
||||
):
|
||||
placement = _placement(ep_rank, "cuda")
|
||||
info = ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=3,
|
||||
ep_dispatch_algorithm="static",
|
||||
expert_location_metadata=placement,
|
||||
all_to_all_ep=True,
|
||||
)
|
||||
# Experts 0..3, the replicated ones, win every token.
|
||||
bias = torch.zeros(_EXPERTS, device="cuda")
|
||||
bias[:4] = 10.0
|
||||
topk = TopK(
|
||||
top_k=16,
|
||||
layer_id=3,
|
||||
renormalize=True,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
correction_bias=bias,
|
||||
routed_scaling_factor=2.5,
|
||||
output_format=TopKOutputFormat.STANDARD,
|
||||
topk_weights_dtype=torch.bfloat16,
|
||||
)
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
logits = torch.randn(tokens, _EXPERTS, device="cuda", generator=generator)
|
||||
hidden = torch.empty(tokens, 8, device="cuda")
|
||||
|
||||
def megamoe_buffers():
|
||||
# MegaMoE's workspace views: FP32 weights, INT32 ids.
|
||||
return (
|
||||
torch.empty(tokens, 16, dtype=torch.float32, device="cuda"),
|
||||
torch.full((tokens, 16), -1, dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
|
||||
logical = megamoe_buffers()
|
||||
topk(hidden, logits, expert_location_dispatch_info=None, out=logical)
|
||||
physical = megamoe_buffers()
|
||||
routed = topk(hidden, logits, expert_location_dispatch_info=info, out=physical)
|
||||
|
||||
assert routed.topk_ids.data_ptr() == physical[1].data_ptr()
|
||||
assert routed.topk_weights.data_ptr() == physical[0].data_ptr()
|
||||
assert torch.equal(physical[0], logical[0])
|
||||
assert torch.equal(physical[1], placement.rank_dispatch_map()[3][logical[1].long()])
|
||||
# Every slot holds the expert the router chose, on the rank that owns it.
|
||||
assert torch.equal(
|
||||
placement.physical_to_logical_map[3][physical[1].long()].to(torch.int32),
|
||||
logical[1],
|
||||
)
|
||||
replicas = physical[1][logical[1] < 4] - logical[1][logical[1] < 4]
|
||||
if ep_rank == 0:
|
||||
assert (replicas == 0).all() # its own slots 0..3
|
||||
elif ep_rank in (16, 19):
|
||||
assert (replicas == _EXPERTS).all() # rank 19's slots 896..899
|
||||
else:
|
||||
assert ((replicas == 0) | (replicas == _EXPERTS)).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device")
|
||||
def test_static_rank_dispatch_map_builds_under_a_cuda_default_device():
|
||||
# Model construction runs with CUDA as the default device; the host-side
|
||||
# nearest-replica table must not pick it up.
|
||||
replicas = torch.tensor([[[0, -1], [1, 4], [2, -1], [3, 5]]], dtype=torch.int64)
|
||||
with torch.device("cuda"):
|
||||
dispatch = compute_logical_to_rank_dispatch_physical_map(
|
||||
logical_to_all_physical_map=replicas,
|
||||
num_physical_experts=6,
|
||||
ep_rank_nodes=[0, 1],
|
||||
ep_rank=1,
|
||||
)
|
||||
assert dispatch.device.type == "cpu"
|
||||
# Rank 1 owns slots 3..5: it keeps its own replicas of experts 1 and 3 and
|
||||
# sends experts 0 and 2 to their only slots.
|
||||
assert dispatch.tolist() == [[0, 4, 2, 3]]
|
||||
|
||||
|
||||
class _DenseDrafterWeights(Eagle3DeepseekV2ForCausalLM):
|
||||
"""The EAGLE3 MLA drafter's real weight path over one dense projection;
|
||||
attention post-processing needs MLA modules this test does not build."""
|
||||
|
||||
def __init__(self, config, mapping):
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.mapping = mapping
|
||||
self.quant_config = None
|
||||
self.model = nn.Module()
|
||||
self.model.midlayer = nn.Module()
|
||||
self.model.midlayer.mlp = nn.Module()
|
||||
self.model.midlayer.mlp.down_proj = nn.Linear(4, 2, bias=False)
|
||||
|
||||
def post_load_weights(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_eagle3_mla_drafter_loads_at_an_ep_size_that_does_not_divide_the_experts():
|
||||
# The drafter's config carries the target's 896 routed experts, but its one
|
||||
# decoder layer is dense: EP20 must not need an expert partition for it.
|
||||
drafter = _DenseDrafterWeights(
|
||||
PretrainedConfig(n_routed_experts=_EXPERTS, num_hidden_layers=1),
|
||||
_mapping(rank=3),
|
||||
)
|
||||
weight = torch.arange(8, dtype=torch.float32).reshape(2, 4)
|
||||
loaded = drafter.load_weights([("layers.0.mlp.down_proj.weight", weight)])
|
||||
assert loaded == {"model.midlayer.mlp.down_proj.weight"}
|
||||
assert torch.equal(drafter.model.midlayer.mlp.down_proj.weight.data, weight)
|
||||
@@ -415,6 +415,7 @@ def test_attn_dp_exchanges_latents_and_returns_reduced_local_rows(
|
||||
routed_hidden=width,
|
||||
top_k=2,
|
||||
topk=topk,
|
||||
expert_dispatch_info=None,
|
||||
experts=SimpleNamespace(
|
||||
plan={"weight_dtype": "nvfp4" if nvfp4 else "unquant"},
|
||||
w13_input_scale_quant=quant_scale,
|
||||
@@ -632,6 +633,7 @@ def test_attn_dp_fused_moe_keeps_inputs_local_and_owns_combine(
|
||||
mapping=SimpleNamespace(world_size=2, attn=SimpleNamespace(dp_rank=0)),
|
||||
stream_fork=SimpleNamespace(scope=scope),
|
||||
topk=mock.Mock(return_value=StandardTopKOutput(weights, ids, None)),
|
||||
expert_dispatch_info=None,
|
||||
gate=mock.Mock(return_value=torch.zeros(rows, 2)),
|
||||
routed_expert_down_proj=mock.Mock(return_value=(latent, None)),
|
||||
experts=SimpleNamespace(
|
||||
|
||||
Reference in New Issue
Block a user