### 0
PYTHONPATH=<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath torchrun
  --nproc-per-node 8 <REPO_ROOT>/tools/convert_hf_to_torch_dist.py
  --spec miles_plugins.models.kimi_k3 get_kimi_k3_spec
  --disable-bias-linear
  --num-layers 4
  --hidden-size 7168
  --ffn-hidden-size 33792
  --num-attention-heads 96
  --num-query-groups 96
  --kv-channels 256
  --normalization RMSNorm
  --position-embedding-type none
  --rotary-base 10000
  --norm-epsilon 1e-5
  --hidden-dropout 0
  --attention-dropout 0
  --disable-bf16-reduced-precision-matmul
  --swiglu
  --untie-embeddings-and-output-weights
  --vocab-size 163840
  --make-vocab-size-divisible-by 128
  --multi-latent-attention
  --q-lora-rank 1536
  --kv-lora-rank 512
  --qk-head-dim 128
  --qk-pos-emb-head-dim 64
  --v-head-dim 128
  --qk-layernorm
  --attention-softmax-in-fp32
  --num-experts 64
  --moe-layer-freq '[0,1,1,1]'
  --moe-ffn-hidden-size 3072
  --moe-latent-size 3584
  --moe-shared-expert-intermediate-size 6144
  --moe-router-topk 16
  --moe-router-score-function sigmoid
  --moe-router-pre-softmax
  --moe-router-enable-expert-bias
  --moe-router-load-balancing-type none
  --moe-token-dispatcher-type alltoall
  --moe-router-bias-update-rate 0
  --moe-aux-loss-coeff 0
  --moe-router-topk-scaling-factor 1.0
  --moe-router-dtype fp32
  --moe-grouped-gemm
  --moe-permute-fusion
  --hf-checkpoint <SANDBOX>/models/Kimi-K3-4layer-64experts-bf16
  --save <SANDBOX>/models/Kimi-K3-4layer-64experts-bf16_torch_dist
  --bf16
  --tensor-model-parallel-size 1
  --pipeline-model-parallel-size 1
  --context-parallel-size 1
  --expert-model-parallel-size 8
  --expert-tensor-parallel-size 1
  --megatron-to-hf-mode raw 
