Files
Xiaoze Fan fd863cce3e feat: add --embed-weights host for a pinned host embedding table (#806)
* feat: add --embed-weights host for a pinned host embedding table

* refactor: check the tie from the model config and share the op walk
2026-10-23 14:41:37 -07:00

64 lines
1.4 KiB
Python

from .activation import (
GATED_ACTIVATIONS,
gated_act_and_mul,
gelu_and_mul,
gelu_tanh_and_mul,
silu_and_mul,
swiglu_clamp_and_mul,
swigluoai_and_mul,
)
from .base import BaseOP, OPList, StateLessOP, iter_ops
from .embedding import ParallelLMHead, VocabParallelEmbedding
from .linear import (
LinearColParallelMerged,
LinearOProj,
LinearQKVMerged,
LinearReplicated,
LinearRowParallel,
)
from .moe import MoELayer, OffloadMoELayer, iter_moe_layers, make_moe_layer
from .norm import (
GatedRMSNorm,
GemmaPlusOneRMSNorm,
GemmaPlusOneRMSNormFused,
GemmaRMSNorm,
LayerNorm,
RMSNorm,
RMSNormFused,
)
from .rotary import get_rope, set_rope_device
__all__ = [
"silu_and_mul",
"gelu_and_mul",
"gelu_tanh_and_mul",
"swigluoai_and_mul",
"swiglu_clamp_and_mul",
"gated_act_and_mul",
"GATED_ACTIVATIONS",
"BaseOP",
"StateLessOP",
"OPList",
"iter_ops",
"VocabParallelEmbedding",
"ParallelLMHead",
"LinearColParallelMerged",
"LinearRowParallel",
"LinearOProj",
"LinearQKVMerged",
"LayerNorm",
"RMSNorm",
"RMSNormFused",
"GatedRMSNorm",
"GemmaRMSNorm",
"GemmaPlusOneRMSNorm",
"GemmaPlusOneRMSNormFused",
"get_rope",
"set_rope_device",
"LinearReplicated",
"MoELayer",
"OffloadMoELayer",
"iter_moe_layers",
"make_moe_layer",
]