Files
Xiaoze Fan 609191d81f feat(gguf): take image input on Qwen and Gemma-4 GGUF checkpoints from an mmproj (#803)
* feat(gguf): load the vision tower from a llama.cpp mmproj

Qwen3.5 / Qwen3.6 / Qwen3.8 (qwen3vl_merger) and Gemma-4 26B-A4B / 31B (gemma4v) / 12B (gemma4uv) take image input from a separate mmproj file: --mmproj, else one in the model's own folder. The tower runs in bf16 (quantized mmproj files are dequantized) and its config comes from the GGUF metadata plus per-arch defaults.

* docs(gguf): describe mmproj image input without a model list

* test(gguf): drop the mmproj tests

* fix(gguf): escape the model folder when globbing for mmproj files

* fix(gguf): skip auto-discovered mmproj files the text model cannot use

* fix(gguf): serve text from a GGUF that carries vision tensors itself
2026-10-22 14:56:19 -07:00

65 lines
1.4 KiB
Python

from .arch import (
is_arch_supported,
is_rocm,
get_rocm_gfx_arch,
is_sm90_family,
is_sm90_supported,
is_sm100_family,
is_sm100_supported,
)
from .hf import (
cached_load_hf_config,
download_hf_weight,
load_eos_token_ids,
load_generation_sampling,
load_image_processor,
load_tokenizer,
load_toolcall_anchor_id,
)
from .logger import init_logger
from .misc import UNSET, Unset, align_ceil, align_down, call_if_main, div_ceil, div_even, mem_GB
from .mp import (
ZmqAsyncPullQueue,
ZmqAsyncPushQueue,
ZmqPubQueue,
ZmqPullQueue,
ZmqPushQueue,
ZmqSubQueue,
)
from .registry import Registry
from .torch_utils import nvtx_annotate, torch_dtype
__all__ = [
"cached_load_hf_config",
"download_hf_weight",
"load_eos_token_ids",
"load_generation_sampling",
"load_tokenizer",
"load_toolcall_anchor_id",
"init_logger",
"is_arch_supported",
"is_rocm",
"get_rocm_gfx_arch",
"is_sm90_family",
"is_sm90_supported",
"is_sm100_family",
"is_sm100_supported",
"call_if_main",
"div_even",
"div_ceil",
"align_ceil",
"align_down",
"mem_GB",
"UNSET",
"Unset",
"torch_dtype",
"nvtx_annotate",
"Registry",
"ZmqPushQueue",
"ZmqPullQueue",
"ZmqPubQueue",
"ZmqSubQueue",
"ZmqAsyncPushQueue",
"ZmqAsyncPullQueue",
]