mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
195 lines
7.6 KiB
Python
195 lines
7.6 KiB
Python
"""Shared GGUF access helpers: detection, metadata, and tensor enumeration.
|
|
|
|
Thin layer over ``gguf.GGUFReader`` (gguf-py). Metadata is read into a plain dict
|
|
keyed by the GGUF field name (``general.architecture``, ``gemma4.block_count`` ...);
|
|
tensors are indexed as ``GgufTensorInfo`` records (ggml dims, ggml type, and where the
|
|
packed bytes sit in the memory-mapped file).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import os
|
|
import re
|
|
import struct
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
|
|
def is_gguf_path(model_path: str) -> bool:
|
|
"""A single ``.gguf`` file (the only GGUF layout FreeToken loads directly)."""
|
|
return isinstance(model_path, str) and os.path.isfile(model_path) and model_path.endswith(
|
|
".gguf"
|
|
)
|
|
|
|
|
|
# Canonical name of the metadata-only GGUF that ``convert_checkpoint`` drops into an FTW
|
|
# dir built from a bare ``.gguf`` source. A GGUF carries its config AND tokenizer in the
|
|
# file's KV section, not sibling files, so a converted checkpoint has nowhere else to read
|
|
# them from -- this file is the header + KV bytes verbatim (tensor_count patched to 0, no
|
|
# tensor infos, no weight data), letting the FTW dir resolve config/tokenizer the exact
|
|
# same way the original ``.gguf`` file does.
|
|
FTW_METADATA_GGUF = "source_metadata.gguf"
|
|
# Records whether the source carried an untied "output.weight" head (the tensor table
|
|
# is stripped from metadata-only gguf files, so the fact travels as a KV).
|
|
OUTPUT_WEIGHT_PRESENT_KV = "freetoken.output_weight_present"
|
|
|
|
|
|
def gguf_config_source(model_path: str) -> str | None:
|
|
"""The ``.gguf`` file to source config/tokenizer/metadata from, or ``None``.
|
|
|
|
A bare ``.gguf`` file resolves to itself; an FTW dir carrying a
|
|
:data:`FTW_METADATA_GGUF` resolves to that embedded metadata file. This is the single
|
|
seam config/tokenizer dispatch uses to decide "this checkpoint is GGUF-config-sourced"
|
|
-- a real file and a converted-FTW dir both land on a genuine ``.gguf`` path the reader
|
|
can parse, so no downstream code learns about the FTW wrapper.
|
|
"""
|
|
if is_gguf_path(model_path):
|
|
return model_path
|
|
if isinstance(model_path, str) and os.path.isdir(model_path):
|
|
cand = os.path.join(model_path, FTW_METADATA_GGUF)
|
|
if os.path.isfile(cand):
|
|
return cand
|
|
return None
|
|
|
|
|
|
def write_metadata_gguf(source_gguf: str, dest_path: str) -> None:
|
|
"""Write a metadata-only GGUF: the source's header + KV section byte-for-byte, with
|
|
``tensor_count`` patched to 0 (no tensor infos, no weight data). Reading only the
|
|
header+KV is cheap; the multi-GB tensor data is never touched.
|
|
|
|
Validates by re-parsing: the copy must list zero tensors and expose the identical KV
|
|
key set (the KV *bytes* are copied verbatim, so identical keys imply identical values).
|
|
"""
|
|
from freetoken.third_party import gguf
|
|
|
|
reader = gguf.GGUFReader(source_gguf)
|
|
assert reader.tensors, f"{source_gguf}: no tensors to bound the KV section"
|
|
# The first tensor-info record starts exactly where the KV section ends (GGUF places no
|
|
# padding between KV and tensor infos; padding is only before the tensor *data*).
|
|
kv_end = int(reader.tensors[0].field.offset)
|
|
buf = bytearray(reader.data[:kv_end].tobytes()) # header + all KV, verbatim
|
|
buf[8:16] = b"\x00" * 8 # tensor_count is a u64 at byte 8; 0 is byte-order agnostic
|
|
# The tensor table is dropped, but config derivation needs one fact from it (an
|
|
# untied output head shows up only as an "output.weight" tensor). Append it as an
|
|
# extra KV and bump kv_count (u64 at byte 16). Little-endian only -- the re-parse
|
|
# below fails loudly on a big-endian source.
|
|
key = OUTPUT_WEIGHT_PRESENT_KV.encode()
|
|
present = any(t.name == "output.weight" for t in reader.tensors)
|
|
buf += struct.pack("<Q", len(key)) + key
|
|
buf += struct.pack("<I", int(gguf.GGUFValueType.BOOL)) + bytes([1 if present else 0])
|
|
struct.pack_into("<Q", buf, 16, struct.unpack_from("<Q", buf, 16)[0] + 1)
|
|
tmp = dest_path + ".tmp"
|
|
with open(tmp, "wb") as f:
|
|
f.write(buf)
|
|
os.replace(tmp, dest_path)
|
|
|
|
check = gguf.GGUFReader(dest_path)
|
|
assert not check.tensors, "metadata gguf still lists tensors after patch"
|
|
src_keys = {k for k in reader.fields if not k.startswith("GGUF.")}
|
|
dst_keys = {k for k in check.fields if not k.startswith("GGUF.")}
|
|
assert dst_keys == src_keys | {OUTPUT_WEIGHT_PRESENT_KV}, (
|
|
f"metadata gguf KV keys differ from source: "
|
|
f"missing {sorted(src_keys - dst_keys)}, extra {sorted(dst_keys - src_keys - {OUTPUT_WEIGHT_PRESENT_KV})}"
|
|
)
|
|
|
|
|
|
def _field_value(reader, name: str) -> Any:
|
|
field = reader.fields.get(name)
|
|
if field is None:
|
|
return None
|
|
return field.contents()
|
|
|
|
|
|
@functools.cache
|
|
def _reader(model_path: str):
|
|
from freetoken.third_party import gguf
|
|
|
|
return gguf.GGUFReader(model_path)
|
|
|
|
|
|
@functools.cache
|
|
def load_gguf_metadata(model_path: str) -> dict[str, Any]:
|
|
"""All GGUF KV metadata as ``{field_name: python_value}`` (arrays -> lists)."""
|
|
reader = _reader(model_path)
|
|
return {name: field.contents() for name, field in reader.fields.items()}
|
|
|
|
|
|
_SPLIT_RE = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")
|
|
|
|
|
|
@functools.cache
|
|
def gguf_split_paths(model_path: str) -> tuple[str, ...]:
|
|
"""Every file of a split GGUF (``name-00001-of-00002.gguf``), in order; a single file is its own list."""
|
|
count = int(_field_value(_reader(model_path), "split.count") or 1)
|
|
if count == 1:
|
|
return (model_path,)
|
|
match = _SPLIT_RE.search(model_path)
|
|
if match is None:
|
|
raise ValueError(f"{model_path} is one of {count} GGUF splits but is not named like name-00001-of-{count:05d}.gguf")
|
|
stem = model_path[: match.start()]
|
|
paths = tuple(f"{stem}-{i:05d}-of-{count:05d}.gguf" for i in range(1, count + 1))
|
|
missing = [p for p in paths if not os.path.isfile(p)]
|
|
if missing:
|
|
raise FileNotFoundError(f"GGUF split files missing: {missing}")
|
|
return paths
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GgufTensorInfo:
|
|
"""Where one tensor's packed bytes sit: ``ne`` is in ggml order (fastest dim first)."""
|
|
|
|
name: str
|
|
path: str
|
|
offset: int
|
|
nbytes: int
|
|
ggml_type: int
|
|
ne: tuple[int, ...]
|
|
# the model path the index was built from, which resolves tensors in any split
|
|
root: str = ""
|
|
|
|
def data(self) -> np.ndarray:
|
|
"""Zero-copy uint8 view of the bytes (memory-mapped)."""
|
|
reader = _reader(self.path)
|
|
return reader.data[self.offset : self.offset + self.nbytes]
|
|
|
|
|
|
@functools.cache
|
|
def gguf_tensor_index(model_path: str) -> dict[str, GgufTensorInfo]:
|
|
"""Every tensor of every split, by name."""
|
|
index: dict[str, GgufTensorInfo] = {}
|
|
for path in gguf_split_paths(model_path):
|
|
for t in _reader(path).tensors:
|
|
index[t.name] = GgufTensorInfo(
|
|
t.name, path, int(t.data_offset), int(t.n_bytes), int(t.tensor_type), tuple(int(x) for x in t.shape), model_path
|
|
)
|
|
return index
|
|
|
|
|
|
def gguf_architecture(model_path: str) -> str:
|
|
arch = _field_value(_reader(model_path), "general.architecture")
|
|
if arch is None:
|
|
raise ValueError(f"GGUF file {model_path} has no general.architecture")
|
|
return str(arch)
|
|
|
|
|
|
def gguf_tensor_names(model_path: str) -> set[str]:
|
|
return {t.name for path in gguf_split_paths(model_path) for t in _reader(path).tensors}
|
|
|
|
|
|
__all__ = [
|
|
"is_gguf_path",
|
|
"FTW_METADATA_GGUF",
|
|
"OUTPUT_WEIGHT_PRESENT_KV",
|
|
"gguf_config_source",
|
|
"write_metadata_gguf",
|
|
"GgufTensorInfo",
|
|
"gguf_split_paths",
|
|
"gguf_tensor_index",
|
|
"load_gguf_metadata",
|
|
"gguf_architecture",
|
|
"gguf_tensor_names",
|
|
]
|