mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat: add --embed-weights host for a pinned host embedding table * refactor: check the tie from the model config and share the op walk
64 lines
1.4 KiB
Python
64 lines
1.4 KiB
Python
from .activation import (
|
|
GATED_ACTIVATIONS,
|
|
gated_act_and_mul,
|
|
gelu_and_mul,
|
|
gelu_tanh_and_mul,
|
|
silu_and_mul,
|
|
swiglu_clamp_and_mul,
|
|
swigluoai_and_mul,
|
|
)
|
|
from .base import BaseOP, OPList, StateLessOP, iter_ops
|
|
from .embedding import ParallelLMHead, VocabParallelEmbedding
|
|
from .linear import (
|
|
LinearColParallelMerged,
|
|
LinearOProj,
|
|
LinearQKVMerged,
|
|
LinearReplicated,
|
|
LinearRowParallel,
|
|
)
|
|
from .moe import MoELayer, OffloadMoELayer, iter_moe_layers, make_moe_layer
|
|
from .norm import (
|
|
GatedRMSNorm,
|
|
GemmaPlusOneRMSNorm,
|
|
GemmaPlusOneRMSNormFused,
|
|
GemmaRMSNorm,
|
|
LayerNorm,
|
|
RMSNorm,
|
|
RMSNormFused,
|
|
)
|
|
from .rotary import get_rope, set_rope_device
|
|
|
|
__all__ = [
|
|
"silu_and_mul",
|
|
"gelu_and_mul",
|
|
"gelu_tanh_and_mul",
|
|
"swigluoai_and_mul",
|
|
"swiglu_clamp_and_mul",
|
|
"gated_act_and_mul",
|
|
"GATED_ACTIVATIONS",
|
|
"BaseOP",
|
|
"StateLessOP",
|
|
"OPList",
|
|
"iter_ops",
|
|
"VocabParallelEmbedding",
|
|
"ParallelLMHead",
|
|
"LinearColParallelMerged",
|
|
"LinearRowParallel",
|
|
"LinearOProj",
|
|
"LinearQKVMerged",
|
|
"LayerNorm",
|
|
"RMSNorm",
|
|
"RMSNormFused",
|
|
"GatedRMSNorm",
|
|
"GemmaRMSNorm",
|
|
"GemmaPlusOneRMSNorm",
|
|
"GemmaPlusOneRMSNormFused",
|
|
"get_rope",
|
|
"set_rope_device",
|
|
"LinearReplicated",
|
|
"MoELayer",
|
|
"OffloadMoELayer",
|
|
"iter_moe_layers",
|
|
"make_moe_layer",
|
|
]
|