"--disable-bias-linear"
"--group-query-attention"
"--num-attention-heads"
"32"
"--num-query-groups"
"2"
"--kv-channels"
"128"
"--num-layers"
"88"
"--hidden-size"
"4096"
"--ffn-hidden-size"
"2688"
"--normalization"
"RMSNorm"
"--position-embedding-type"
"none"
"--vocab-size"
"131072"
"--make-vocab-size-divisible-by"
"128"
"--untie-embeddings-and-output-weights"
"--num-experts"
"512"
"--moe-router-topk"
"22"
"--moe-ffn-hidden-size"
"2688"
"--moe-shared-expert-intermediate-size"
"5376"
"--moe-latent-size"
"1024"
"--moe-router-score-function"
"sigmoid"
"--moe-router-enable-expert-bias"
"--moe-grouped-gemm"
"--moe-router-dtype"
"fp32"
"--moe-router-num-groups"
"1"
"--moe-router-group-topk"
"1"
"--moe-router-topk-scaling-factor"
"5.0"
"--moe-router-pre-softmax"
"--moe-router-load-balancing-type"
"seq_aux_loss"
"--moe-router-bias-update-rate"
"0"
"--moe-aux-loss-coeff"
"0"
