mirror of
https://github.com/radixark/miles.git
synced 2026-11-11 09:59:33 +08:00
maocheng23
ca8645ceb9
* retool in one commit * clean up as comments * clean up as comments * clean up as comments
32 lines
694 B
Python
32 lines
694 B
Python
from datasets import load_dataset
|
|
|
|
ds = load_dataset("JoeYing/ReTool-SFT")["train"]
|
|
|
|
|
|
def convert(sample):
|
|
conversations = sample["messages"]
|
|
|
|
def convert_role(role):
|
|
if role == "user":
|
|
return "user"
|
|
elif role == "assistant":
|
|
return "assistant"
|
|
elif role == "system":
|
|
return "system"
|
|
else:
|
|
raise ValueError(f"Unknown role: {role}")
|
|
|
|
messages = [
|
|
{
|
|
"role": convert_role(turn["role"]),
|
|
"content": turn["content"],
|
|
}
|
|
for turn in conversations
|
|
]
|
|
|
|
return {"messages": messages}
|
|
|
|
|
|
ds = ds.map(convert)
|
|
ds.to_parquet("./data/retool/ReTool-SFT.parquet")
|