from __future__ import annotations import glob import json import os import re import struct from dataclasses import dataclass from typing import Iterable, Iterator import torch from freetoken.utils import div_ceil, download_hf_weight SPLIT_DIM_0 = (".q_proj", ".k_proj", ".v_proj", ".gate_proj", ".up_proj") SPLIT_DIM_1 = (".o_proj", ".down_proj") @dataclass(frozen=True) class MergeRule: fused_suffix: str slot: str slots: tuple[str, ...] def shard_tensor( key: str, value: torch.Tensor, *, rank: int, world_size: int, num_kv_heads: int, ) -> torch.Tensor: if any(key.count(sub) for sub in SPLIT_DIM_0): is_kv_proj = any(key.count(sub) for sub in (".k_proj", ".v_proj")) if is_kv_proj and num_kv_heads is not None and num_kv_heads < world_size: head_dim = value.shape[0] // num_kv_heads head_idx = rank * num_kv_heads // world_size return value[head_idx * head_dim : (head_idx + 1) * head_dim].clone() return value.chunk(world_size, dim=0)[rank].clone() if any(key.count(sub) for sub in SPLIT_DIM_1): return value.chunk(world_size, dim=1)[rank].clone() if key.count("lm_head") or key.count("embed_tokens"): num_embeddings = value.shape[0] num_embeddings_per_partition = div_ceil(num_embeddings, world_size) vocab_start_idx = rank * num_embeddings_per_partition vocab_end_idx = min((rank + 1) * num_embeddings_per_partition, num_embeddings) return value[vocab_start_idx:vocab_end_idx, :].clone() return value def iter_weight_files(model_path: str) -> list[str]: model_folder = download_hf_weight(model_path) # The index is authoritative: download_hf_weight only fetches the shards it names, and it # covers subdirectory layouts (nvidia's FP4-Local releases ship tensors/model-*.safetensors). index = os.path.join(model_folder, "model.safetensors.index.json") if os.path.exists(index): with open(index, encoding="utf-8") as f: shards = sorted(set(json.load(f)["weight_map"].values())) return [os.path.join(model_folder, shard) for shard in shards] files = glob.glob(f"{model_folder}/*.safetensors") return [f for f in files if not f.endswith("consolidated.safetensors")] or files def safetensors_weight_map(folder: str) -> dict[str, str]: """Tensor name -> shard basename, from the index or from each shard's header when the checkpoint ships none.""" index = os.path.join(folder, "model.safetensors.index.json") if os.path.exists(index): with open(index, encoding="utf-8") as f: return json.load(f)["weight_map"] weight_map: dict[str, str] = {} for path in sorted(iter_weight_files(folder)): with open(path, "rb") as fh: n = struct.unpack(" None: """drop a file's page cache: banks + full checkpoint cache don't both fit in host RAM (OOM).""" try: fd = os.open(path, os.O_RDONLY) try: os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) finally: os.close(fd) except OSError: pass def iter_root_safetensor_files_from_index( model_path: str, *, index_file: str = "model.safetensors.index.json", ) -> list[str]: model_folder = download_hf_weight(model_path) if os.path.basename(os.path.normpath(model_folder)) in {"metal", "original"}: raise ValueError("GPT-OSS loading requires the root GPT-OSS model directory") root_files = sorted(glob.glob(os.path.join(model_folder, "*.safetensors"))) index_path = os.path.join(model_folder, index_file) if not os.path.isfile(index_path): files = [f for f in root_files if not f.endswith("consolidated.safetensors")] if not files: raise ValueError("No root GPT-OSS safetensors shards found") return files with open(index_path, encoding="utf-8") as f: weight_map = json.load(f)["weight_map"] indexed_files = [] for filename in dict.fromkeys(weight_map.values()): if os.path.dirname(filename): continue path = os.path.join(model_folder, filename) if path in root_files: indexed_files.append(path) if not indexed_files: raise ValueError( "No root GPT-OSS safetensors shards found from model.safetensors.index.json" ) return sorted(indexed_files) def _merge_info(key: str, rules: dict[str, MergeRule]) -> tuple[str, MergeRule] | None: for suffix, rule in rules.items(): if key.endswith(suffix + ".weight") or key.endswith(suffix) or key.count(suffix): return key.replace(suffix, rule.fused_suffix), rule return None def iter_merged_tensors( tensors: Iterable[tuple[str, torch.Tensor]], rules: dict[str, MergeRule], *, model_name: str, ) -> Iterator[tuple[str, torch.Tensor]]: merge_buf: dict[str, dict[str, torch.Tensor]] = {} for name, tensor in tensors: info = _merge_info(name, rules) if info is None: yield name, tensor continue merged_key, rule = info slots = merge_buf.setdefault(merged_key, {}) slots[rule.slot] = tensor if not all(slot in slots for slot in rule.slots): continue parts = [slots[slot] for slot in rule.slots] del merge_buf[merged_key] yield merged_key, torch.cat(parts, dim=0) assert not merge_buf, ( f"{model_name}: Incomplete merge groups in checkpoint: {list(merge_buf.keys())}" ) # --------------------------------------------------------------------------------- # compressed-tensors NVFP4 (llm-compressor) dense-weight helpers, shared by the # models that serve such checkpoints natively (muse_glimmer). Storage: # ``weight_packed`` (uint8 [O, IN//2]) + ``weight_scale`` (fp8-e4m3 block [O, IN//16]) # + a scalar ``weight_global_scale``. The stored global is the *quant-side* scale, so # the dequant/native global is its reciprocal (vLLM inverts it identically). # --------------------------------------------------------------------------------- # Quant scales consumed with their ``weight_packed``. CT_SCALE_SUFFIXES = ( ".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale", ) class ShardReader: """Serves tensors by name across safetensors shards (handles opened lazily). The quant scales of a ``weight_packed`` can land in a DIFFERENT shard than the packed weight itself (Muse-Glimmer-30B-NVFP4 splits layer 49's down_proj across the shard boundary), so sibling lookups must go through the index's weight_map rather than the shard the packed weight came from.""" def __init__(self, model_path: str, device: torch.device): folder = download_hf_weight(model_path) self._map = {name: os.path.join(folder, shard) for name, shard in safetensors_weight_map(folder).items()} self._device = str(device) self._handles: dict[str, object] = {} def files(self) -> list[str]: return sorted(set(self._map.values())) def has(self, name: str) -> bool: return name in self._map def names_in(self, file: str) -> list[str]: return [name for name, shard in self._map.items() if shard == file] def get_tensor(self, name: str) -> torch.Tensor: import safetensors file = self._map[name] h = self._handles.get(file) if h is None: h = safetensors.safe_open(file, framework="pt", device=self._device).__enter__() self._handles[file] = h return h.get_tensor(name) def close(self) -> None: for h in self._handles.values(): try: h.__exit__(None, None, None) except Exception: # noqa: BLE001 -- best-effort handle cleanup pass self._handles.clear() def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]: """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp32 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method.""" w = f.get_tensor(raw_base + ".weight_packed") s = f.get_tensor(raw_base + ".weight_scale") wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32) g = (1.0 / wg).expand(w.shape[0]).contiguous() a = None if f.has(raw_base + ".input_global_scale"): a = (1.0 / f.get_tensor(raw_base + ".input_global_scale").reshape(()).to(torch.float32)) return w, s, g, a def ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict, groups: dict[str, tuple[str, ...]]): """Buffer a native NVFP4 fusion part ``(w, s, g, a)``; once complete, emit the concatenated ``.weight`` / ``.weight_scale`` / ``.weight_global`` (output-dim concat, each part keeps its own scales, so the fused FP4 weight is exact) plus the largest ``.input_scale`` when every part has one. ``[]`` while incomplete, ``None`` if ``base`` is not a fusion part of any group in ``groups``.""" for fused_suffix, parts in groups.items(): for idx, part in enumerate(parts): if base.endswith(part): key = base[: -len(part)] + fused_suffix slots = buf.setdefault(key, {}) slots[idx] = parts_tuple if len(slots) < len(parts): return [] del buf[key] ws = [slots[i][0] for i in range(len(parts))] ss = [slots[i][1] for i in range(len(parts))] gs = [slots[i][2] for i in range(len(parts))] acts = [slots[i][3] for i in range(len(parts))] out = [ (key + ".weight", torch.cat(ws, dim=0)), (key + ".weight_scale", torch.cat(ss, dim=0)), (key + ".weight_global", torch.cat(gs, dim=0)), ] if all(a is not None for a in acts): out.append((key + ".input_scale", torch.stack(acts).max())) return out return None def ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict[str, tuple[str, ...]]): """Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]`` while incomplete, ``None`` if ``base`` is not a part of any group in ``groups``.""" for fused_suffix, parts in groups.items(): for idx, part in enumerate(parts): if base.endswith(part): key = base[: -len(part)] + fused_suffix slots = buf.setdefault(key, {}) slots[idx] = tensor if len(slots) < len(parts): return [] del buf[key] return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))] return None def _expert_stack_info(key: str, expert_pattern: re.Pattern[str]) -> tuple[str, int] | None: match = expert_pattern.match(key) if match is None: return None packed_name = match.group("name") if packed_name.endswith(".weight"): packed_name = packed_name.removesuffix(".weight") return f"{match.group('prefix')}.{packed_name}", int(match.group("idx")) def iter_stacked_experts( tensors: Iterable[tuple[str, torch.Tensor]], *, num_experts: int, model_name: str, expert_pattern: re.Pattern[str], ) -> Iterator[tuple[str, torch.Tensor]]: expert_buf: dict[str, dict[int, torch.Tensor]] = {} for name, tensor in tensors: expert_info = _expert_stack_info(name, expert_pattern) if expert_info is None: yield name, tensor continue packed_key, expert_idx = expert_info slots = expert_buf.setdefault(packed_key, {}) slots[expert_idx] = tensor if len(slots) != num_experts: continue experts = [slots[idx] for idx in range(num_experts)] del expert_buf[packed_key] yield packed_key, torch.stack(experts, dim=0) assert not expert_buf, ( f"{model_name}: Incomplete expert tensors in checkpoint: {list(expert_buf.keys())}" ) __all__ = [ "MergeRule", "iter_root_safetensor_files_from_index", "iter_weight_files", "safetensors_weight_map", "iter_merged_tensors", "iter_stacked_experts", "shard_tensor", ]