"--spec"
"miles_plugins.models.glm5.glm5"
"get_glm5_spec"
"--moe-layer-freq"
"[0]*3+[1]*17"
"--num-experts"
"256"
"--moe-shared-expert-intermediate-size"
"2048"
"--moe-router-topk"
"8"
"--moe-grouped-gemm"
"--moe-permute-fusion"
"--moe-ffn-hidden-size"
"2048"
"--moe-router-score-function"
"sigmoid"
"--moe-router-pre-softmax"
"--moe-router-enable-expert-bias"
"--moe-router-bias-update-rate"
"0"
"--moe-router-load-balancing-type"
"seq_aux_loss"
"--moe-router-topk-scaling-factor"
"2.5"
"--moe-aux-loss-coeff"
"0"
"--moe-router-dtype"
"fp32"
"--make-vocab-size-divisible-by"
"16"
"--num-layers"
"20"
"--hidden-size"
"6144"
"--ffn-hidden-size"
"12288"
"--num-attention-heads"
"64"
"--disable-bias-linear"
"--swiglu"
"--untie-embeddings-and-output-weights"
"--position-embedding-type"
"rope"
"--no-position-embedding"
"--normalization"
"RMSNorm"
"--qk-layernorm"
"--multi-latent-attention"
"--q-lora-rank"
"2048"
"--kv-lora-rank"
"512"
"--qk-head-dim"
"192"
"--v-head-dim"
"256"
"--kv-channels"
"192"
"--qk-pos-emb-head-dim"
"64"
"--vocab-size"
"154880"
"--rotary-base"
"1000000"
"--enable-experimental"
