import copy import functools import json import os from typing import Any, FrozenSet, Mapping from huggingface_hub import hf_hub_download, snapshot_download from huggingface_hub.utils import EntryNotFoundError from tqdm.asyncio import tqdm from transformers import ( AutoConfig, AutoTokenizer, GenerationConfig, PretrainedConfig, PreTrainedTokenizerBase, ) from transformers.utils import SAFE_WEIGHTS_INDEX_NAME from freetoken.utils.logger import init_logger logger = init_logger(__name__) class DisabledTqdm(tqdm): def __init__(self, *args, **kwargs): kwargs.pop("name", None) kwargs["disable"] = True super().__init__(*args, **kwargs) def load_tokenizer(model_path: str) -> PreTrainedTokenizerBase: from freetoken.models.gguf.reader import gguf_config_source if (gguf_src := gguf_config_source(model_path)) is not None: from freetoken.models.gguf.tokenizer import load_gguf_tokenizer return load_gguf_tokenizer(gguf_src) tokenizer = AutoTokenizer.from_pretrained(model_path) # Some Mistral models store chat_template in a separate JSON file if not getattr(tokenizer, "chat_template", None): try: path = hf_hub_download(repo_id=model_path, filename="chat_template.json") with open(path, "r", encoding="utf-8") as f: tokenizer.chat_template = json.load(f)["chat_template"] except Exception: pass return tokenizer def load_image_processor(model_path: str) -> Any: """The checkpoint's image processor; a GGUF builds it from the preprocessor_config its family derives.""" from freetoken.models.gguf.reader import gguf_config_source if gguf_config_source(model_path) is not None: from transformers.models.auto.image_processing_auto import get_image_processor_class_from_name config = cached_load_hf_config(model_path).preprocessor_config if config is None: raise ValueError(f"{model_path}: GGUF has no vision mmproj to build an image processor from") return get_image_processor_class_from_name(config["image_processor_type"]).from_dict(config) from transformers import AutoImageProcessor return AutoImageProcessor.from_pretrained(model_path) def load_eos_token_ids( model_path: str, tokenizer: PreTrainedTokenizerBase ) -> FrozenSet[int]: """Return the full set of stop-token ids for generation. Chat models often terminate a turn with a token other than ``tokenizer.eos_token`` (e.g. Gemma uses ````), and list every stop id in ``generation_config.json``'s ``eos_token_id``. Honor that list (unioned with the tokenizer's eos) so generation halts correctly instead of running to ``max_tokens``. """ from freetoken.models.gguf.reader import gguf_config_source if (gguf_src := gguf_config_source(model_path)) is not None: from freetoken.models.gguf.tokenizer import gguf_eos_token_ids return frozenset(gguf_eos_token_ids(gguf_src)) ids: set[int] = set() if tokenizer.eos_token_id is not None: ids.add(int(tokenizer.eos_token_id)) try: gen_eos = GenerationConfig.from_pretrained(model_path).eos_token_id except Exception: gen_eos = None if isinstance(gen_eos, int): ids.add(gen_eos) elif isinstance(gen_eos, (list, tuple)): ids.update(int(x) for x in gen_eos) return frozenset(ids) def load_toolcall_anchor_id( tokenizer: PreTrainedTokenizerBase, opener: str | None ) -> int | None: """The single token id of ``opener`` -- the wire format's unique tool-call opening marker, declared by the model's detector (``BaseFormatDetector.toolcall_opener``). None when there is no opener or the tokenizer spells it with more than one token: the scheduler's special-token checkpoint matches sampled ids one at a time, so only a one-token opener can anchor.""" if not opener: return None try: ids = tokenizer.encode(opener, add_special_tokens=False) except Exception: return None return int(ids[0]) if len(ids) == 1 else None def load_generation_sampling(model_path: str) -> dict[str, Any]: """Recommended sampling defaults from ``generation_config.json`` (sglang's ``sampling_defaults='model'``). Returns ``{temperature, top_k, top_p}`` for the keys present; if the model recommends greedy (``do_sample=false``) returns greedy (``temperature=0``). Empty dict if there is no generation config / no sampling info. Many reasoning models (e.g. Qwen3.5: temp 1.0, top_k 20, top_p 0.95) ship these here; applying them avoids the greedy/unfiltered repetition loops these models fall into. """ # GGUF carries the recommended sampling in metadata (general.sampling.*), not a # generation_config.json -- read it directly so --sampling-defaults=model works. from freetoken.models.gguf.reader import gguf_config_source if (gguf_src := gguf_config_source(model_path)) is not None: from freetoken.models.gguf.reader import load_gguf_metadata meta = load_gguf_metadata(gguf_src) out: dict[str, Any] = {} if (v := meta.get("general.sampling.temp")) is not None: out["temperature"] = float(v) if (v := meta.get("general.sampling.top_k")) is not None: out["top_k"] = int(v) if (v := meta.get("general.sampling.top_p")) is not None: out["top_p"] = float(v) return out try: gc = GenerationConfig.from_pretrained(model_path) except Exception: return {} if getattr(gc, "do_sample", None) is False: return {"temperature": 0.0} out: dict[str, Any] = {} for key in ("temperature", "top_k", "top_p"): val = getattr(gc, key, None) if val is not None: out[key] = val return out class RawConfigShim: """Attribute view over a checkpoint's raw ``config.json``. Fallback for model types NEWER than the installed transformers (AutoConfig raises on an unknown ``model_type`` when the checkpoint ships no ``auto_map``, e.g. Muse-Glimmer on transformers < 5.15). FreeToken only reads config fields (parse_config) and never instantiates modeling code, so the raw JSON is enough. ``*_config`` sub-dicts are wrapped for attribute access, matching PretrainedConfig's nested-config behavior; every other value is served raw (``rope_parameters`` and friends stay plain dicts, as they do on the real class). """ def __init__(self, data: dict | None = None, **kwargs: Any) -> None: self._data = {**(data or {}), **kwargs} def __getattr__(self, name: str) -> Any: if name == "_name_or_path": # PretrainedConfig carries the checkpoint path under this name and # DSV4's parse_config reads it (to find inference/config.json), so it # must survive the underscore guard below. return self.__dict__.get("_data", {}).get("_name_or_path", "") if name.startswith("_"): raise AttributeError(name) try: value = self._data[name] except KeyError: raise AttributeError(name) from None if name.endswith("_config") and isinstance(value, dict): return RawConfigShim(value) return value def to_dict(self) -> dict: return json.loads(json.dumps(self._data)) # deep copy, callers may mutate def optional_hf_file(model_path: str, filename: str) -> str | None: """Local path of ``filename`` in a checkpoint dir or Hub repo; None when the checkpoint has no such file.""" if os.path.isdir(model_path): path = os.path.join(model_path, filename) return path if os.path.isfile(path) else None try: return hf_hub_download(repo_id=model_path, filename=filename) except EntryNotFoundError: return None def _raw_config_json(model_path: str) -> dict: if os.path.isdir(model_path): path = os.path.join(model_path, "config.json") else: path = hf_hub_download(repo_id=model_path, filename="config.json") with open(path, encoding="utf-8") as f: return json.load(f) def sidecar_quantization_config(model_path: str) -> dict | None: """The ``quantization_config`` an old ModelOpt export keeps only in ``hf_quant_config.json``, or None.""" sidecar = optional_hf_file(model_path, "hf_quant_config.json") if sidecar is None: return None with open(sidecar, encoding="utf-8") as f: quant = json.load(f).get("quantization") if not isinstance(quant, dict): return None return {"quant_method": "modelopt", **quant} def _merge_sidecar_quantization_config(config: Any, model_path: str) -> None: # config.json wins when it has one; the sidecar is only read for exports that never wrote it there if getattr(config, "quantization_config", None) is not None: return if getattr(getattr(config, "text_config", None), "quantization_config", None) is not None: return quant = sidecar_quantization_config(model_path) if quant is None: return if isinstance(config, RawConfigShim): config._data["quantization_config"] = quant else: config.quantization_config = quant @functools.cache def _load_hf_config(model_path: str) -> Any: # trust_remote_code: checkpoints that ship a custom config class via ``auto_map`` # (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields # (parse_config) and never instantiates the checkpoint's modeling code. try: config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) except ValueError as exc: # Unknown model_type on this transformers version: serve off the raw JSON. # Anything else (bad path, malformed JSON) stays fatal. if "model type" not in str(exc): raise config = RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path) _merge_sidecar_quantization_config(config, model_path) return config def _with_overrides(config: Any, data: dict, overrides: Mapping[str, Any]) -> dict: merged = dict(data) for key, value in overrides.items(): section = getattr(config, key, None) if isinstance(value, Mapping) and isinstance(section, (PretrainedConfig, RawConfigShim)): merged[key] = _with_overrides(section, merged.get(key) or {}, value) else: merged[key] = copy.deepcopy(value) return merged def cached_load_hf_config( model_path: str, overrides: Mapping[str, Any] | None = None ) -> PretrainedConfig: """A fresh copy of the checkpoint's config with ``overrides`` applied as vLLM's --hf-overrides does: nested config sections update key by key, every other value is replaced whole.""" # A .gguf file (or an FTW dir converted from one) carries its own metadata (no HF # config.json); return a shim the model registry dispatches on instead of a # PretrainedConfig. from freetoken.models.gguf.reader import gguf_config_source if (gguf_src := gguf_config_source(model_path)) is not None: if overrides: raise ValueError("--hf-overrides applies to a HF config.json; a GGUF carries its own metadata") from freetoken.models.gguf.config import build_gguf_shim return build_gguf_shim(gguf_src) config = _load_hf_config(model_path) data = config.to_dict() if overrides: data = _with_overrides(config, data, overrides) if isinstance(config, RawConfigShim): return RawConfigShim(data) return type(config)(**data) def _weight_allow_patterns(repo_id: str) -> list[str]: try: index = hf_hub_download(repo_id, SAFE_WEIGHTS_INDEX_NAME, tqdm_class=DisabledTqdm) with open(index, encoding="utf-8") as f: shards = sorted(set(json.load(f)["weight_map"].values())) except Exception as e: logger.warning( "no usable %s for %s (%s); falling back to *.safetensors", SAFE_WEIGHTS_INDEX_NAME, repo_id, e, ) return ["*.safetensors"] return shards or ["*.safetensors"] def download_hf_weight(model_path: str) -> str: if os.path.isdir(model_path): return model_path try: return snapshot_download( model_path, allow_patterns=_weight_allow_patterns(model_path), tqdm_class=DisabledTqdm, ) except Exception as e: raise ValueError( f"Model path '{model_path}' is neither a local directory nor a valid model ID: {e}" )