# returncode: 0

### 0
"pkill"
"-9"
"sglang"

### 1
"sleep"
"3"

### 2
"ray"
"stop"
"--force"

### 3
"pkill"
"-9"
"ray"

### 4
"pkill"
"-9"
"python"

### 5
"sleep"
"3"

### 6
"pkill"
"-9"
"ray"

### 7
"pkill"
"-9"
"python"

### 8
"pkill"
"-9"
"redis"

### 9
"python3"
"<REPO_ROOT>/miles/utils/external_utils/model_args_utils.py"
"glm5-744B-A40B"

### 10
"ray"
"start"
"--head"
"--node-ip-address"
"10.0.0.1"
"--num-gpus"
"8"
"--disable-usage-stats"
"--dashboard-host=0.0.0.0"
"--dashboard-port=8265"

### 11
"python3"
"-c"
"import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()"

### 12
"mkdir"
"-p"
"<SANDBOX>/workdir"

### 13
"rm"
"-f"
"<SANDBOX>/workdir/job_done_p2p"

### 14
"ray"
"job"
"submit"
"--address=http://127.0.0.1:8265"
"--runtime-env-json={\n  \"env_vars\": {\n    \"MC_TRANSFER_TIMEOUT\": \"600\",\n    \"RAY_DEBUG\": \"1\",\n    \"PYTHONPATH\": \"/root/Megatron-LM/\",\n    \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",\n    \"NCCL_NVLS_ENABLE\": \"1\",\n    \"INDEXER_ROPE_NEOX_STYLE\": \"0\",\n    \"NVSHMEM_DISABLE_NCCL\": \"1\",\n    \"MILES_LOG_DIR\": \"<SANDBOX>/workdir\"\n  }\n}"
"--"
"python3"
"train.py"
"--spec"
"miles_plugins.models.glm5.glm5"
"get_glm5_spec"
"--moe-layer-freq"
"[0]*3+[1]*75"
"--num-experts"
"256"
"--moe-shared-expert-intermediate-size"
"2048"
"--moe-router-topk"
"8"
"--moe-grouped-gemm"
"--moe-permute-fusion"
"--moe-ffn-hidden-size"
"2048"
"--moe-router-score-function"
"sigmoid"
"--moe-router-pre-softmax"
"--moe-router-enable-expert-bias"
"--moe-router-bias-update-rate"
"0"
"--moe-router-load-balancing-type"
"seq_aux_loss"
"--moe-router-topk-scaling-factor"
"2.5"
"--moe-aux-loss-coeff"
"0"
"--moe-router-dtype"
"fp32"
"--make-vocab-size-divisible-by"
"16"
"--num-layers"
"78"
"--hidden-size"
"6144"
"--ffn-hidden-size"
"12288"
"--num-attention-heads"
"64"
"--disable-bias-linear"
"--swiglu"
"--untie-embeddings-and-output-weights"
"--position-embedding-type"
"rope"
"--no-position-embedding"
"--normalization"
"RMSNorm"
"--qk-layernorm"
"--multi-latent-attention"
"--q-lora-rank"
"2048"
"--kv-lora-rank"
"512"
"--qk-head-dim"
"192"
"--v-head-dim"
"256"
"--kv-channels"
"192"
"--qk-pos-emb-head-dim"
"64"
"--vocab-size"
"154880"
"--rotary-base"
"1000000"
"--enable-experimental"
"--hf-checkpoint"
"/root/models/GLM-5"
"--ref-load"
"/root/GLM-5_torch_dist"
"--prompt-data"
"/root/datasets/dapo-math-17k/dapo-math-17k.jsonl"
"--input-key"
"prompt"
"--label-key"
"label"
"--apply-chat-template"
"--rollout-shuffle"
"--rm-type"
"deepscaler"
"--num-rollout"
"13"
"--rollout-batch-size"
"4"
"--n-samples-per-prompt"
"4"
"--rollout-max-response-len"
"100"
"--rollout-temperature"
"1"
"--global-batch-size"
"16"
"--optimizer"
"adam"
"--lr"
"1e-6"
"--lr-decay-style"
"constant"
"--weight-decay"
"0.1"
"--adam-beta1"
"0.9"
"--adam-beta2"
"0.98"
"--optimizer-cpu-offload"
"--overlap-cpu-optimizer-d2h-h2d"
"--use-precision-aware-optimizer"
"--advantage-estimator"
"grpo"
"--kl-loss-coef"
"0.00"
"--kl-loss-type"
"low_var_kl"
"--kl-coef"
"0.00"
"--entropy-coef"
"0.00"
"--eps-clip"
"0.2"
"--eps-clip-high"
"0.28"
"--tensor-model-parallel-size"
"4"
"--sequence-parallel"
"--pipeline-model-parallel-size"
"8"
"--context-parallel-size"
"2"
"--expert-model-parallel-size"
"16"
"--expert-tensor-parallel-size"
"1"
"--decoder-last-pipeline-num-layers"
"8"
"--recompute-granularity"
"full"
"--recompute-method"
"uniform"
"--recompute-num-layers"
"1"
"--use-dynamic-batch-size"
"--max-tokens-per-gpu"
"256"
"--data-pad-size-multiplier"
"4096"
"--log-probs-chunk-size"
"1024"
"--rollout-num-gpus-per-engine"
"64"
"--rollout-num-gpus"
"128"
"--sglang-mem-fraction-static"
"0.90"
"--sglang-enable-dp-attention"
"--sglang-ep-size"
"64"
"--sglang-dp-size"
"64"
"--sglang-moe-dense-tp-size"
"1"
"--sglang-enable-dp-lm-head"
"--sglang-page-size"
"64"
"--sglang-nsa-decode-backend"
"flashmla_sparse"
"--sglang-nsa-prefill-backend"
"flashmla_sparse"
"--sglang-attention-backend"
"nsa"
"--sglang-cuda-graph-max-bs"
"8"
"--sglang-max-running-requests"
"512"
"--sglang-chunked-prefill-size"
"131072"
"--sglang-watchdog-timeout"
"3600"
"--sglang-disable-cuda-graph"
"--sglang-remote-instance-weight-loader-start-seed-via-transfer-engine"
"--sglang-remote-instance-weight-loader-ib-device"
"{\"0\":\"ibp0\",\"1\":\"ibp1\",\"2\":\"ibp2\",\"3\":\"ibp3\",\"4\":\"ibp4\",\"5\":\"ibp5\",\"6\":\"ibp6\",\"7\":\"ibp7\"}"
"--sglang-model-loader-extra-config"
"{\"enable_multithread_load\":true,\"num_threads\":8}"
"--attention-dropout"
"0.0"
"--hidden-dropout"
"0.0"
"--accumulate-allreduce-grads-in-fp32"
"--attention-softmax-in-fp32"
"--attention-backend"
"flash"
"--allgather-cp"
"--moe-token-dispatcher-type"
"alltoall"
"--actor-num-nodes"
"16"
"--actor-num-gpus-per-node"
"8"
"--update-weight-buffer-size"
"2147483648"
"--check-weight-update-equal"
"--update-weight-transfer-mode"
"p2p"
