Files
FreeToken/python/freetoken/utils/hf.py

318 lines
13 KiB
Python

import copy
import functools
import json
import os
from typing import Any, FrozenSet, Mapping
from huggingface_hub import hf_hub_download, snapshot_download
from huggingface_hub.utils import EntryNotFoundError
from tqdm.asyncio import tqdm
from transformers import (
AutoConfig,
AutoTokenizer,
GenerationConfig,
PretrainedConfig,
PreTrainedTokenizerBase,
)
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from freetoken.utils.logger import init_logger
logger = init_logger(__name__)
class DisabledTqdm(tqdm):
def __init__(self, *args, **kwargs):
kwargs.pop("name", None)
kwargs["disable"] = True
super().__init__(*args, **kwargs)
def load_tokenizer(model_path: str) -> PreTrainedTokenizerBase:
from freetoken.models.gguf.reader import gguf_config_source
if (gguf_src := gguf_config_source(model_path)) is not None:
from freetoken.models.gguf.tokenizer import load_gguf_tokenizer
return load_gguf_tokenizer(gguf_src)
tokenizer = AutoTokenizer.from_pretrained(model_path)
# Some Mistral models store chat_template in a separate JSON file
if not getattr(tokenizer, "chat_template", None):
try:
path = hf_hub_download(repo_id=model_path, filename="chat_template.json")
with open(path, "r", encoding="utf-8") as f:
tokenizer.chat_template = json.load(f)["chat_template"]
except Exception:
pass
return tokenizer
def load_image_processor(model_path: str) -> Any:
"""The checkpoint's image processor; a GGUF builds it from the preprocessor_config its family derives."""
from freetoken.models.gguf.reader import gguf_config_source
if gguf_config_source(model_path) is not None:
from transformers.models.auto.image_processing_auto import get_image_processor_class_from_name
config = cached_load_hf_config(model_path).preprocessor_config
if config is None:
raise ValueError(f"{model_path}: GGUF has no vision mmproj to build an image processor from")
return get_image_processor_class_from_name(config["image_processor_type"]).from_dict(config)
from transformers import AutoImageProcessor
return AutoImageProcessor.from_pretrained(model_path)
def load_eos_token_ids(
model_path: str, tokenizer: PreTrainedTokenizerBase
) -> FrozenSet[int]:
"""Return the full set of stop-token ids for generation.
Chat models often terminate a turn with a token other than ``tokenizer.eos_token``
(e.g. Gemma uses ``<end_of_turn>``), and list every stop id in
``generation_config.json``'s ``eos_token_id``. Honor that list (unioned with the
tokenizer's eos) so generation halts correctly instead of running to ``max_tokens``.
"""
from freetoken.models.gguf.reader import gguf_config_source
if (gguf_src := gguf_config_source(model_path)) is not None:
from freetoken.models.gguf.tokenizer import gguf_eos_token_ids
return frozenset(gguf_eos_token_ids(gguf_src))
ids: set[int] = set()
if tokenizer.eos_token_id is not None:
ids.add(int(tokenizer.eos_token_id))
try:
try:
gen_eos = GenerationConfig.from_pretrained(model_path).eos_token_id
except OSError: # no generation_config.json, config.json can still list the stop ids
gen_eos = getattr(_load_hf_config(model_path), "eos_token_id", None)
except Exception:
gen_eos = None
if isinstance(gen_eos, int):
ids.add(gen_eos)
elif isinstance(gen_eos, (list, tuple)):
ids.update(int(x) for x in gen_eos)
return frozenset(ids)
def load_toolcall_anchor_id(
tokenizer: PreTrainedTokenizerBase, opener: str | None
) -> int | None:
"""The single token id of ``opener`` -- the wire format's unique tool-call opening
marker, declared by the model's detector (``BaseFormatDetector.toolcall_opener``).
None when there is no opener or the tokenizer spells it with more than one token:
the scheduler's special-token checkpoint matches sampled ids one at a time, so only
a one-token opener can anchor."""
if not opener:
return None
try:
ids = tokenizer.encode(opener, add_special_tokens=False)
except Exception:
return None
return int(ids[0]) if len(ids) == 1 else None
def load_generation_sampling(model_path: str) -> dict[str, Any]:
"""Recommended sampling defaults from ``generation_config.json`` (sglang's
``sampling_defaults='model'``). Returns ``{temperature, top_k, top_p}`` for the keys
present; if the model recommends greedy (``do_sample=false``) returns greedy
(``temperature=0``). Empty dict if there is no generation config / no sampling info.
Many reasoning models (e.g. Qwen3.5: temp 1.0, top_k 20, top_p 0.95) ship these here;
applying them avoids the greedy/unfiltered repetition loops these models fall into.
"""
# GGUF carries the recommended sampling in metadata (general.sampling.*), not a
# generation_config.json -- read it directly so --sampling-defaults=model works.
from freetoken.models.gguf.reader import gguf_config_source
if (gguf_src := gguf_config_source(model_path)) is not None:
from freetoken.models.gguf.reader import load_gguf_metadata
meta = load_gguf_metadata(gguf_src)
out: dict[str, Any] = {}
if (v := meta.get("general.sampling.temp")) is not None:
out["temperature"] = float(v)
if (v := meta.get("general.sampling.top_k")) is not None:
out["top_k"] = int(v)
if (v := meta.get("general.sampling.top_p")) is not None:
out["top_p"] = float(v)
return out
try:
gc = GenerationConfig.from_pretrained(model_path)
except Exception:
return {}
if getattr(gc, "do_sample", None) is False:
return {"temperature": 0.0}
out: dict[str, Any] = {}
for key in ("temperature", "top_k", "top_p"):
val = getattr(gc, key, None)
if val is not None:
out[key] = val
return out
class RawConfigShim:
"""Attribute view over a checkpoint's raw ``config.json``.
Fallback for model types NEWER than the installed transformers (AutoConfig raises
on an unknown ``model_type`` when the checkpoint ships no ``auto_map``, e.g.
Muse-Glimmer on transformers < 5.15). FreeToken only reads config fields
(parse_config) and never instantiates modeling code, so the raw JSON is enough.
``*_config`` sub-dicts are wrapped for attribute access, matching
PretrainedConfig's nested-config behavior; every other value is served raw
(``rope_parameters`` and friends stay plain dicts, as they do on the real class).
"""
def __init__(self, data: dict | None = None, **kwargs: Any) -> None:
self._data = {**(data or {}), **kwargs}
def __getattr__(self, name: str) -> Any:
if name == "_name_or_path":
# PretrainedConfig carries the checkpoint path under this name and
# DSV4's parse_config reads it (to find inference/config.json), so it
# must survive the underscore guard below.
return self.__dict__.get("_data", {}).get("_name_or_path", "")
if name.startswith("_"):
raise AttributeError(name)
try:
value = self._data[name]
except KeyError:
raise AttributeError(name) from None
if name.endswith("_config") and isinstance(value, dict):
return RawConfigShim(value)
return value
def to_dict(self) -> dict:
return json.loads(json.dumps(self._data)) # deep copy, callers may mutate
def optional_hf_file(model_path: str, filename: str) -> str | None:
"""Local path of ``filename`` in a checkpoint dir or Hub repo; None when the checkpoint has no such file."""
if os.path.isdir(model_path):
path = os.path.join(model_path, filename)
return path if os.path.isfile(path) else None
try:
return hf_hub_download(repo_id=model_path, filename=filename)
except EntryNotFoundError:
return None
def _raw_config_json(model_path: str) -> dict:
if os.path.isdir(model_path):
path = os.path.join(model_path, "config.json")
else:
path = hf_hub_download(repo_id=model_path, filename="config.json")
with open(path, encoding="utf-8") as f:
return json.load(f)
def sidecar_quantization_config(model_path: str) -> dict | None:
"""The ``quantization_config`` an old ModelOpt export keeps only in ``hf_quant_config.json``, or None."""
sidecar = optional_hf_file(model_path, "hf_quant_config.json")
if sidecar is None:
return None
with open(sidecar, encoding="utf-8") as f:
quant = json.load(f).get("quantization")
if not isinstance(quant, dict):
return None
return {"quant_method": "modelopt", **quant}
def _merge_sidecar_quantization_config(config: Any, model_path: str) -> None:
# config.json wins when it has one; the sidecar is only read for exports that never wrote it there
if getattr(config, "quantization_config", None) is not None:
return
if getattr(getattr(config, "text_config", None), "quantization_config", None) is not None:
return
quant = sidecar_quantization_config(model_path)
if quant is None:
return
if isinstance(config, RawConfigShim):
config._data["quantization_config"] = quant
else:
config.quantization_config = quant
@functools.cache
def _load_hf_config(model_path: str) -> Any:
# trust_remote_code: checkpoints that ship a custom config class via ``auto_map``
# (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields
# (parse_config) and never instantiates the checkpoint's modeling code.
try:
config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
except ValueError as exc:
# Unknown model_type on this transformers version: serve off the raw JSON.
# Anything else (bad path, malformed JSON) stays fatal.
if "model type" not in str(exc):
raise
config = RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)
_merge_sidecar_quantization_config(config, model_path)
return config
def _with_overrides(config: Any, data: dict, overrides: Mapping[str, Any]) -> dict:
merged = dict(data)
for key, value in overrides.items():
section = getattr(config, key, None)
if isinstance(value, Mapping) and isinstance(section, (PretrainedConfig, RawConfigShim)):
merged[key] = _with_overrides(section, merged.get(key) or {}, value)
else:
merged[key] = copy.deepcopy(value)
return merged
def cached_load_hf_config(
model_path: str, overrides: Mapping[str, Any] | None = None
) -> PretrainedConfig:
"""A fresh copy of the checkpoint's config with ``overrides`` applied as vLLM's --hf-overrides
does: nested config sections update key by key, every other value is replaced whole."""
# A .gguf file (or an FTW dir converted from one) carries its own metadata (no HF
# config.json); return a shim the model registry dispatches on instead of a
# PretrainedConfig.
from freetoken.models.gguf.reader import gguf_config_source
if (gguf_src := gguf_config_source(model_path)) is not None:
if overrides:
raise ValueError("--hf-overrides applies to a HF config.json; a GGUF carries its own metadata")
from freetoken.models.gguf.config import build_gguf_shim
return build_gguf_shim(gguf_src)
config = _load_hf_config(model_path)
data = config.to_dict()
if overrides:
data = _with_overrides(config, data, overrides)
if isinstance(config, RawConfigShim):
return RawConfigShim(data)
return type(config)(**data)
def _weight_allow_patterns(repo_id: str) -> list[str]:
try:
index = hf_hub_download(repo_id, SAFE_WEIGHTS_INDEX_NAME, tqdm_class=DisabledTqdm)
with open(index, encoding="utf-8") as f:
shards = sorted(set(json.load(f)["weight_map"].values()))
except Exception as e:
logger.warning(
"no usable %s for %s (%s); falling back to *.safetensors",
SAFE_WEIGHTS_INDEX_NAME, repo_id, e,
)
return ["*.safetensors"]
return shards or ["*.safetensors"]
def download_hf_weight(model_path: str) -> str:
if os.path.isdir(model_path):
return model_path
try:
return snapshot_download(
model_path,
allow_patterns=_weight_allow_patterns(model_path),
tqdm_class=DisabledTqdm,
)
except Exception as e:
raise ValueError(
f"Model path '{model_path}' is neither a local directory nor a valid model ID: {e}"
)