mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(gguf): load the vision tower from a llama.cpp mmproj Qwen3.5 / Qwen3.6 / Qwen3.8 (qwen3vl_merger) and Gemma-4 26B-A4B / 31B (gemma4v) / 12B (gemma4uv) take image input from a separate mmproj file: --mmproj, else one in the model's own folder. The tower runs in bf16 (quantized mmproj files are dequantized) and its config comes from the GGUF metadata plus per-arch defaults. * docs(gguf): describe mmproj image input without a model list * test(gguf): drop the mmproj tests * fix(gguf): escape the model folder when globbing for mmproj files * fix(gguf): skip auto-discovered mmproj files the text model cannot use * fix(gguf): serve text from a GGUF that carries vision tensors itself
65 lines
1.4 KiB
Python
65 lines
1.4 KiB
Python
from .arch import (
|
|
is_arch_supported,
|
|
is_rocm,
|
|
get_rocm_gfx_arch,
|
|
is_sm90_family,
|
|
is_sm90_supported,
|
|
is_sm100_family,
|
|
is_sm100_supported,
|
|
)
|
|
from .hf import (
|
|
cached_load_hf_config,
|
|
download_hf_weight,
|
|
load_eos_token_ids,
|
|
load_generation_sampling,
|
|
load_image_processor,
|
|
load_tokenizer,
|
|
load_toolcall_anchor_id,
|
|
)
|
|
from .logger import init_logger
|
|
from .misc import UNSET, Unset, align_ceil, align_down, call_if_main, div_ceil, div_even, mem_GB
|
|
from .mp import (
|
|
ZmqAsyncPullQueue,
|
|
ZmqAsyncPushQueue,
|
|
ZmqPubQueue,
|
|
ZmqPullQueue,
|
|
ZmqPushQueue,
|
|
ZmqSubQueue,
|
|
)
|
|
from .registry import Registry
|
|
from .torch_utils import nvtx_annotate, torch_dtype
|
|
|
|
__all__ = [
|
|
"cached_load_hf_config",
|
|
"download_hf_weight",
|
|
"load_eos_token_ids",
|
|
"load_generation_sampling",
|
|
"load_tokenizer",
|
|
"load_toolcall_anchor_id",
|
|
"init_logger",
|
|
"is_arch_supported",
|
|
"is_rocm",
|
|
"get_rocm_gfx_arch",
|
|
"is_sm90_family",
|
|
"is_sm90_supported",
|
|
"is_sm100_family",
|
|
"is_sm100_supported",
|
|
"call_if_main",
|
|
"div_even",
|
|
"div_ceil",
|
|
"align_ceil",
|
|
"align_down",
|
|
"mem_GB",
|
|
"UNSET",
|
|
"Unset",
|
|
"torch_dtype",
|
|
"nvtx_annotate",
|
|
"Registry",
|
|
"ZmqPushQueue",
|
|
"ZmqPullQueue",
|
|
"ZmqPubQueue",
|
|
"ZmqSubQueue",
|
|
"ZmqAsyncPushQueue",
|
|
"ZmqAsyncPullQueue",
|
|
]
|