mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* feat(rotary): size the rope table for YaRN and add --hf-overrides
* fix(rotary): guard mrope's table width and copy --hf-overrides values
- MRotaryEmbedding asserts its cos/sin table is rotary_dim wide. A proportional
rope's table spans the head, and the mrope kernel indexes the rotary_dim // 2
section table with the table's width.
- _with_overrides deep-copies every value it stores: transformers fills rope
defaults into the rope_parameters it is handed, which was the caller's dict.
- The rope-table tests the engine test covers fold into it, the two shared-table
tests are one, and the --hf-overrides test moves to tests/utils/test_hf.py,
where it loads the override against the same values written into config.json,
for a model type transformers knows and one it does not.
- docs/cli.md lists --hf-overrides.
* fix(server): give --hf-overrides a fresh dict per parse
argparse hands its default object to every parse that omits the flag, so
default={} put one shared dict in every ServerArgs. The flag defaults to None and
becomes a new dict where the arguments are assembled, as --mm-processor-kwargs
does.
* docs(rotary): an indexer's heads differ in width from the attention's, not wider
35 lines
1.7 KiB
Python
35 lines
1.7 KiB
Python
"""cached_load_hf_config with --hf-overrides, against the same config written into config.json."""
|
|
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from freetoken.utils.hf import cached_load_hf_config
|
|
|
|
|
|
def _checkpoint(path, model_type: str, dtype: str, rope_parameters: dict) -> str:
|
|
path.mkdir()
|
|
text_config = {"hidden_size": 64, "max_position_embeddings": 4096, "rope_parameters": rope_parameters}
|
|
(path / "config.json").write_text(json.dumps({"model_type": model_type, "dtype": dtype, "text_config": text_config}))
|
|
return str(path)
|
|
|
|
|
|
@pytest.mark.parametrize("model_type", ["qwen3_5", "a_model_type_transformers_does_not_know"])
|
|
def test_hf_overrides_load_as_if_the_checkpoint_config_said_so(tmp_path, model_type):
|
|
# vLLM's merge: a nested config section updates key by key, any other value is replaced whole
|
|
yarn = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 4096}
|
|
checkpoint = _checkpoint(tmp_path / "checkpoint", model_type, "bfloat16", {"rope_type": "default", "rope_theta": 1e6})
|
|
edited = _checkpoint(tmp_path / "edited", model_type, "float16", yarn)
|
|
overrides = {"dtype": "float16", "text_config": {"rope_parameters": yarn}}
|
|
requested = copy.deepcopy(overrides)
|
|
|
|
hf = cached_load_hf_config(checkpoint, overrides)
|
|
expected = cached_load_hf_config(edited)
|
|
assert hf.dtype == expected.dtype
|
|
assert hf.text_config.to_dict() == expected.text_config.to_dict()
|
|
# transformers fills rope defaults into the rope_parameters it is handed
|
|
assert overrides == requested
|
|
# a fresh copy each time: the override never reaches the cached checkpoint config
|
|
assert cached_load_hf_config(checkpoint).text_config.rope_parameters["rope_type"] == "default"
|