mirror of
https://github.com/radixark/miles.git
synced 2026-11-06 08:43:11 +08:00
afd088cb6b
Co-authored-by: Jiajun Li <jiajun.li@radixark.ai> Co-authored-by: guapisolo <guapisolo@gmail.com>
32 lines
1.2 KiB
YAML
32 lines
1.2 KiB
YAML
eval:
|
|
defaults:
|
|
n_samples_per_eval_prompt: 1
|
|
temperature: 0.6
|
|
top_p: 0.95
|
|
top_k: -1
|
|
max_response_len: 24576
|
|
datasets: # these eval tasks go through miles dataset config and default rollout function (miles.rollout.inference_rollout.inference_rollout_common.InferenceRolloutFn)
|
|
- name: gpqa # hf download --repo-type dataset zyzshishui0627/gpqa_diamond --local-dir /root/gpqa
|
|
path: /root/gpqa/gpqa_eval.jsonl
|
|
rm_type: gpqa
|
|
n_samples_per_eval_prompt: 2
|
|
- name: ifbench # hf download --repo-type dataset zyzshishui0627/IFBench --local-dir /root/ifbench
|
|
path: /root/ifbench/IFBench_eval.jsonl
|
|
rm_type: ifbench
|
|
n_samples_per_eval_prompt: 1
|
|
delegate: # these tasks go through delegate eval function (examples.experimental.eval.eval_delegate_rollout.generate_rollout)
|
|
- name: skills
|
|
# this url should align with env docker network alias
|
|
url: http://skills_server:9050/evaluate
|
|
timeout_secs: 7200
|
|
max_retries: 5
|
|
headers: {}
|
|
datasets:
|
|
- name: aime25
|
|
max_response_len: 8192
|
|
n_samples_per_eval_prompt: 8
|
|
- name: arena-hard
|
|
n_samples_per_eval_prompt: 2
|
|
- name: hle
|
|
max_response_len: 32768
|