trending/Model-Optimizer
1
0
Fork 0
mirror of https://github.com/NVIDIA/Model-Optimizer.git synced 2026-10-02 03:14:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove Qwen tokenizer modification (#390)

Browse Source 
Signed-off-by: Chenjie Luo <chenjiel@nvidia.com>
This commit is contained in:
Chenjie Luo
2025-10-04 17:41:08 +05:30
committed by GitHub
parent adcb1a1c1c
commit 340eb7a756
5 changed files with 0 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/llm_eval/gen_model_answer.py
-8
View File
@@ -180,14 +180,6 @@ def get_model_answers(
# Model Optimizer modification
tokenizer = get_tokenizer(model_path, trust_remote_code=args.trust_remote_code)
if checkpoint_dir:
# get model type
last_part = os.path.basename(checkpoint_dir)
model_type = last_part.split("_")[0]
# Some models require to set pad_token and eos_token based on external config (e.g., qwen)
if model_type == "qwen":
tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
assert LLM is not None, "tensorrt_llm APIs could not be imported."
model = LLM(checkpoint_dir, tokenizer=tokenizer)
elif not nim_model:
examples/llm_eval/mmlu.py
-8
View File
@@ -253,14 +253,6 @@ def main(
model_path = kwargs["model_path"]
tokenizer = get_tokenizer(model_path, trust_remote_code=kwargs.get("trust_remote_code", False))
if kwargs.get("checkpoint_dir"):
# get model type
last_part = os.path.basename(kwargs["checkpoint_dir"])
model_type = last_part.split("_")[0]
# Some models require to set pad_token and eos_token based on external config (e.g., qwen)
if model_type == "qwen":
tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
assert LLM is not None, "tensorrt_llm APIs could not be imported."
medusa_choices = kwargs.get("medusa_choices")
model = LLM(
examples/llm_eval/quantization_utils.py
-4
View File
@@ -53,10 +53,6 @@ def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, trust_remote_code=False):
padding_side="left",
trust_remote_code=trust_remote_code,
)
if type(tokenizer).__name__ == "QWenTokenizer":
# qwen use token id 151643 as pad and eos tokens
tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
# can't set attribute 'pad_token' for "<unk>"
if tokenizer.pad_token != "<unk>":
examples/llm_ptq/example_utils.py
-5
View File
@@ -54,11 +54,6 @@ def get_tokenizer(ckpt_path, trust_remote_code=False, **kwargs):
ckpt_path, trust_remote_code=trust_remote_code, **kwargs
)
if "qwen" in type(tokenizer).__name__.lower():
# qwen use token id 151643 as pad and eos tokens
tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
# can't set attribute 'pad_token' for "<unk>"
# We skip this step for Nemo models
if tokenizer.pad_token != "<unk>" or tokenizer.pad_token is None:
examples/windows/accuracy_benchmark/quantization_utils.py
-4
View File
@@ -37,10 +37,6 @@ def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, trust_remote_code=False):
padding_side="left",
trust_remote_code=trust_remote_code,
)
if type(tokenizer).__name__ == "QWenTokenizer":
# qwen use token id 151643 as pad and eos tokens
tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
# can't set attribute 'pad_token' for "<unk>"
if tokenizer.pad_token != "<unk>":