"--num-layers"
"24"
"--hidden-size"
"2880"
"--ffn-hidden-size"
"2880"
"--num-attention-heads"
"64"
"--group-query-attention"
"--num-query-groups"
"8"
"--kv-channels"
"64"
"--use-rotary-position-embeddings"
"--rotary-percent"
"1.0"
"--rotary-base"
"150000"
"--max-position-embeddings"
"131072"
"--normalization"
"RMSNorm"
"--norm-epsilon"
"1e-5"
"--swiglu"
"--untie-embeddings-and-output-weights"
"--vocab-size"
"201088"
"--softmax-type"
"learnable"
"--window-size"
"128,0"
"--window-attn-skip-freq"
"2"
"--no-masked-softmax-fusion"
"--no-rope-fusion"
"--num-experts"
"32"
"--moe-router-topk"
"4"
"--moe-aux-loss-coeff"
"0.0"
"--moe-token-dispatcher-type"
"alltoall"
"--moe-router-dtype"
"fp32"
"--moe-grouped-gemm"
