"--swiglu"
"--num-layers"
"36"
"--hidden-size"
"4096"
"--ffn-hidden-size"
"11008"
"--num-attention-heads"
"32"
"--group-query-attention"
"--num-query-groups"
"8"
"--use-rotary-position-embeddings"
"--disable-bias-linear"
"--add-qkv-bias"
"--normalization"
"RMSNorm"
"--norm-epsilon"
"1e-05"
"--rotary-base"
"640000"
"--vocab-size"
"151680"
"--untie-embeddings-and-output-weights"
"--max-position-embeddings"
"32768"
"--mtp-num-layers"
"1"
