### 0
PYTHONPATH=<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath torchrun
  --nproc-per-node 8 <REPO_ROOT>/tools/convert_hf_to_torch_dist.py
  --disable-bias-linear
  --num-layers 43
  --hidden-size 4096
  --ffn-hidden-size 2048
  --num-attention-heads 64
  --normalization RMSNorm
  --position-embedding-type rope
  --norm-epsilon 1e-6
  --swiglu
  --untie-embeddings-and-output-weights
  --vocab-size 129280
  --hidden-dropout 0.0
  --attention-dropout 0.0
  --multi-latent-attention
  --q-lora-rank 1024
  --kv-lora-rank 512
  --qk-head-dim 512
  --qk-pos-emb-head-dim 64
  --v-head-dim 512
  --qk-layernorm
  --rotary-scaling-factor 16
  --rotary-base 10000
  --original-max-position-embeddings 65536
  --beta-fast 32
  --beta-slow 1
  --attention-softmax-in-fp32
  --no-rope-fusion
  --num-experts 256
  --moe-layer-freq '[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'
  --moe-ffn-hidden-size 2048
  --moe-router-topk 6
  --moe-shared-expert-intermediate-size 2048
  --moe-router-pre-softmax
  --moe-router-score-function sqrtsoftplus
  --moe-router-enable-expert-bias
  --moe-router-load-balancing-type seq_aux_loss
  --moe-token-dispatcher-type alltoall
  --moe-aux-loss-coeff 0
  --moe-grouped-gemm
  --moe-router-topk-scaling-factor 1.5
  --num-residual-streams 4
  --mhc-sinkhorn-iterations 20
  --csa-compress-ratios '[0,0,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,128,4,0]'
  --csa-compress-rotary-base 160000
  --o-groups 8
  --o-lora-rank 1024
  --moe-n-hash-layers 3
  --csa-window-size 128
  --dsa-indexer-n-heads 64
  --dsa-indexer-head-dim 128
  --dsa-indexer-topk 512
  --spec miles_plugins.models.deepseek_v4.deepseek_v4 get_dsv4_spec
  --activation-func-clamp-value 10
  --no-bias-swiglu-fusion
  --no-activation-func-clamp-shared-expert
  --hf-checkpoint /root/models/DeepSeek-V4-Flash-FP8-bf16
  --save /root/models/DeepSeek-V4-Flash-FP8_torch_dist
  --dsv4-impl miles
  --expert-tensor-parallel-size 1
  --context-parallel-size 1
  --tensor-model-parallel-size 1
  --pipeline-model-parallel-size 1
  --expert-model-parallel-size 8 
