mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* fix(models): find index-only checkpoints that keep shards in a subdirectory * fix(models): make the checkpoint index authoritative for shard discovery The index is read directly so safetensors_weight_map stays the only header-based reader, and _host_ram_fits_parallel sizes the shards the reader will actually open, so the low-RAM serial fallback fires again for subdirectory layouts. * fix(models): fall back to root shards when the index is unusable Share one index parser between iter_weight_files and safetensors_weight_map instead of a third copy; warn and glob the root when the index is broken, absent or empty; reject index keys that escape the model folder; let the hub download fetch the index itself; tell the truth in the diagnose docstring. * revert(models): back out the defensive index fallback, keep the no-index test Per review: a broken index is a broken checkpoint, so the loader keeps the plain one rule -- use the index if there is one, otherwise glob the root. The no-index fallback test stays as the only coverage of the root glob and the consolidated.safetensors rule.
294 lines
10 KiB
Python
294 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
|
|
def test_shard_tensor_splits_vocab_with_ceil_partition():
|
|
from freetoken.models.loader import shard_tensor
|
|
|
|
value = torch.arange(5 * 2, dtype=torch.float32).reshape(5, 2)
|
|
|
|
rank0 = shard_tensor(
|
|
"model.embed_tokens.weight",
|
|
value,
|
|
rank=0,
|
|
world_size=2,
|
|
num_kv_heads=1,
|
|
)
|
|
rank1 = shard_tensor(
|
|
"model.embed_tokens.weight",
|
|
value,
|
|
rank=1,
|
|
world_size=2,
|
|
num_kv_heads=1,
|
|
)
|
|
|
|
assert rank0.tolist() == value[:3].tolist()
|
|
assert rank1.tolist() == value[3:5].tolist()
|
|
|
|
|
|
def test_iter_weight_files_reads_shards_from_an_indexed_subdirectory(tmp_path):
|
|
"""FP4-Local releases keep shards in tensors/; the root glob alone finds nothing."""
|
|
import json
|
|
import os
|
|
from freetoken.models.loader import iter_weight_files
|
|
|
|
sub = tmp_path / "tensors"
|
|
sub.mkdir()
|
|
for i in (1, 2):
|
|
(sub / f"model-{i:05d}-of-00002.safetensors").write_bytes(b"")
|
|
(tmp_path / "model.safetensors.index.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"weight_map": {
|
|
"model.embed_tokens.weight": "tensors/model-00001-of-00002.safetensors",
|
|
"lm_head.weight": "tensors/model-00002-of-00002.safetensors",
|
|
}
|
|
}
|
|
)
|
|
)
|
|
|
|
files = iter_weight_files(str(tmp_path))
|
|
|
|
assert len(files) == 2
|
|
assert all(os.path.exists(f) for f in files)
|
|
|
|
|
|
def test_iter_weight_files_lets_the_index_decide_which_shards_load(tmp_path):
|
|
"""The index is authoritative: it may name root and subdirectory shards; an unlisted root file is not loaded."""
|
|
import json
|
|
from freetoken.models.loader import iter_weight_files
|
|
|
|
root = tmp_path / "model-00001-of-00002.safetensors"
|
|
root.write_bytes(b"")
|
|
(tmp_path / "unlisted.safetensors").write_bytes(b"")
|
|
sub = tmp_path / "tensors"
|
|
sub.mkdir()
|
|
(sub / "model-00002-of-00002.safetensors").write_bytes(b"")
|
|
(tmp_path / "model.safetensors.index.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"weight_map": {
|
|
"model.embed_tokens.weight": "model-00001-of-00002.safetensors",
|
|
"lm_head.weight": "tensors/model-00002-of-00002.safetensors",
|
|
}
|
|
}
|
|
)
|
|
)
|
|
|
|
files = iter_weight_files(str(tmp_path))
|
|
|
|
assert files == [str(root), str(sub / "model-00002-of-00002.safetensors")]
|
|
|
|
|
|
def test_iter_weight_files_falls_back_to_root_shards_without_an_index(tmp_path):
|
|
"""No index: the root glob loads, excluding consolidated.safetensors unless it is the only shard."""
|
|
from freetoken.models.loader import iter_weight_files
|
|
|
|
a = tmp_path / "model-00001-of-00002.safetensors"
|
|
b = tmp_path / "model-00002-of-00002.safetensors"
|
|
consolidated = tmp_path / "consolidated.safetensors"
|
|
for path in (a, b, consolidated):
|
|
path.write_bytes(b"")
|
|
|
|
assert sorted(iter_weight_files(str(tmp_path))) == sorted([str(a), str(b)])
|
|
|
|
a.unlink()
|
|
b.unlink()
|
|
assert iter_weight_files(str(tmp_path)) == [str(consolidated)]
|
|
|
|
|
|
def test_iter_root_safetensor_files_from_index_keeps_only_root_index_shards(tmp_path):
|
|
import json
|
|
from freetoken.models.loader import iter_root_safetensor_files_from_index
|
|
|
|
root_a = tmp_path / "model-00000-of-00002.safetensors"
|
|
root_b = tmp_path / "model-00001-of-00002.safetensors"
|
|
ignored = tmp_path / "consolidated.safetensors"
|
|
metal = tmp_path / "metal" / "model.safetensors"
|
|
original = tmp_path / "original" / "model.safetensors"
|
|
for path in (root_a, root_b, ignored, metal, original):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(b"")
|
|
(tmp_path / "model.safetensors.index.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"weight_map": {
|
|
"model.embed_tokens.weight": root_a.name,
|
|
"lm_head.weight": root_b.name,
|
|
"bad": "metal/model.safetensors",
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
files = iter_root_safetensor_files_from_index(str(tmp_path))
|
|
|
|
assert files == [str(root_a), str(root_b)]
|
|
|
|
|
|
def test_iter_root_safetensor_files_from_index_rejects_subdirectory_model_path(tmp_path):
|
|
import pytest
|
|
from freetoken.models.loader import iter_root_safetensor_files_from_index
|
|
|
|
subdir = tmp_path / "original"
|
|
subdir.mkdir()
|
|
|
|
with pytest.raises(ValueError, match="root GPT-OSS"):
|
|
iter_root_safetensor_files_from_index(str(subdir))
|
|
|
|
|
|
def test_iter_root_safetensor_files_from_index_requires_root_shards(tmp_path):
|
|
import pytest
|
|
from freetoken.models.loader import iter_root_safetensor_files_from_index
|
|
|
|
with pytest.raises(ValueError, match="No root GPT-OSS safetensors shards"):
|
|
iter_root_safetensor_files_from_index(str(tmp_path))
|
|
|
|
|
|
def test_shard_tensor_replicates_kv_heads_when_tp_exceeds_kv_heads():
|
|
from freetoken.models.loader import shard_tensor
|
|
|
|
value = torch.arange(2 * 4, dtype=torch.float32).reshape(2, 4)
|
|
|
|
rank0 = shard_tensor(
|
|
"model.layers.0.self_attn.k_proj.weight",
|
|
value,
|
|
rank=0,
|
|
world_size=4,
|
|
num_kv_heads=2,
|
|
)
|
|
rank1 = shard_tensor(
|
|
"model.layers.0.self_attn.k_proj.weight",
|
|
value,
|
|
rank=1,
|
|
world_size=4,
|
|
num_kv_heads=2,
|
|
)
|
|
rank2 = shard_tensor(
|
|
"model.layers.0.self_attn.k_proj.weight",
|
|
value,
|
|
rank=2,
|
|
world_size=4,
|
|
num_kv_heads=2,
|
|
)
|
|
rank3 = shard_tensor(
|
|
"model.layers.0.self_attn.k_proj.weight",
|
|
value,
|
|
rank=3,
|
|
world_size=4,
|
|
num_kv_heads=2,
|
|
)
|
|
|
|
assert rank0.tolist() == value[:1].tolist()
|
|
assert rank1.tolist() == value[:1].tolist()
|
|
assert rank2.tolist() == value[1:2].tolist()
|
|
assert rank3.tolist() == value[1:2].tolist()
|
|
|
|
|
|
def test_merge_stream_buffers_qkv_until_all_slots_arrive():
|
|
from freetoken.models.loader import MergeRule, iter_merged_tensors
|
|
|
|
tensors = [
|
|
("model.layers.0.self_attn.q_proj.weight", torch.full((1, 2), 1.0)),
|
|
("model.layers.0.self_attn.v_proj.weight", torch.full((1, 2), 3.0)),
|
|
("model.layers.0.self_attn.k_proj.weight", torch.full((1, 2), 2.0)),
|
|
]
|
|
rules = {
|
|
".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
|
|
".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
|
|
".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
|
|
}
|
|
|
|
merged = list(iter_merged_tensors(tensors, rules, model_name="test"))
|
|
|
|
assert len(merged) == 1
|
|
assert merged[0][0] == "model.layers.0.self_attn.qkv_proj.weight"
|
|
assert merged[0][1].tolist() == [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]]
|
|
|
|
|
|
def test_iter_merged_tensors_reports_incomplete_merge_with_model_name():
|
|
import pytest
|
|
from freetoken.models.loader import MergeRule, iter_merged_tensors
|
|
|
|
rules = {
|
|
".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
|
|
".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
|
|
".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
|
|
}
|
|
|
|
with pytest.raises(AssertionError, match="test.*Incomplete merge groups"):
|
|
list(iter_merged_tensors([("x.q_proj.weight", torch.zeros(1, 1))], rules, model_name="test"))
|
|
|
|
|
|
def test_stack_expert_tensors_after_all_experts_arrive():
|
|
from freetoken.models.loader import iter_stacked_experts
|
|
|
|
expert_pattern = re.compile(
|
|
r"^(?P<prefix>.+\.experts)\.(?P<idx>\d+)\.(?P<name>.+)$"
|
|
)
|
|
tensors = [
|
|
("model.layers.0.mlp.experts.1.gate_up_proj.weight", torch.full((1, 2), 11.0)),
|
|
("model.layers.0.mlp.experts.0.gate_up_proj.weight", torch.full((1, 2), 10.0)),
|
|
]
|
|
|
|
packed = list(
|
|
iter_stacked_experts(
|
|
tensors,
|
|
num_experts=2,
|
|
model_name="qwen3_moe",
|
|
expert_pattern=expert_pattern,
|
|
)
|
|
)
|
|
|
|
assert len(packed) == 1
|
|
assert packed[0][0] == "model.layers.0.mlp.experts.gate_up_proj"
|
|
assert packed[0][1].shape == (2, 1, 2)
|
|
assert packed[0][1][0].tolist() == [[10.0, 10.0]]
|
|
assert packed[0][1][1].tolist() == [[11.0, 11.0]]
|
|
|
|
|
|
def test_stacked_expert_pieces_pair_each_layer_in_arrival_order():
|
|
from freetoken.moe.expert_pieces import stacked_expert_pieces
|
|
|
|
config = SimpleNamespace(num_layers=2, num_experts=2)
|
|
tensors = [
|
|
("model.layers.1.mlp.experts.down_proj", torch.full((2, 4, 3), 11.0)),
|
|
("model.layers.0.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 2.0)),
|
|
("model.layers.1.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 3.0)),
|
|
("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),
|
|
]
|
|
|
|
pieces = list(stacked_expert_pieces(tensors, config))
|
|
|
|
assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(1, 0, 2), (0, 0, 2)]
|
|
assert torch.equal(pieces[0][3]["gate_up"], torch.full((2, 3, 4), 3.0))
|
|
assert torch.equal(pieces[0][3]["down"], torch.full((2, 4, 3), 11.0))
|
|
assert torch.equal(pieces[1][3]["gate_up"], torch.full((2, 3, 4), 2.0))
|
|
with pytest.raises(ValueError, match="Missing MoE expert source layers"):
|
|
list(stacked_expert_pieces(tensors[:3], config))
|
|
|
|
|
|
def test_stacked_expert_pieces_skip_the_layers_of_other_formats():
|
|
from freetoken.moe.expert_pieces import stacked_expert_pieces
|
|
|
|
config = SimpleNamespace(num_layers=3, num_experts=2)
|
|
tensors = [
|
|
("model.layers.0.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 2.0)),
|
|
("model.layers.1.mlp.experts.0.gate_proj.weight_scale", torch.zeros(3, 1)),
|
|
("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),
|
|
("model.layers.2.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 3.0)),
|
|
("model.layers.2.mlp.experts.down_proj", torch.full((2, 4, 3), 11.0)),
|
|
]
|
|
|
|
pieces = list(stacked_expert_pieces(tensors, config, layers=[0]))
|
|
|
|
assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(0, 0, 2)]
|
|
with pytest.raises(ValueError, match="Unexpected expert weight key"):
|
|
list(stacked_expert_pieces(tensors, config))
|