Files
Xiaoze Fanandzihaomu e5d3bd3faf feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding

* build(deps): vendor the gguf-py reader

* feat(quant): add the GGUF quant kind and an embedding quant method

* feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path

* fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA

* perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16

* build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small

* fix(gguf): read F32, F16 and BF16 PLE tables without ggml

* fix(moe): pad the offload slot cache for ggml MMQ tail reads

* fix(gguf): support hip builds and runtime devices

* fix(gguf): copy same-type dequantize and check for missing ggml converters

* fix(gguf): load quantized qwen4exp ple_key and ple_value

* feat(gguf): run float parts of a fused GGUF linear as bf16 segments

* fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights

* fix(moe): zero the prefill staging area after the last prefill layer

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-15 16:33:16 -07:00

40 lines
1.4 KiB
Python

"""One QuantConfig per checkpoint dialect; importing the package registers them all."""
from .base import NoQuantConfig, QuantConfig, quantization_config_of
from . import compressed_tensors, fp8, gguf, modelopt, mxfp4
from .compressed_tensors import CompressedTensorsConfig
from .fp8 import Fp8BlockConfig
from .modelopt import ModelOptConfig
from .mxfp4 import Mxfp4Config
from .gguf import GGUFConfig
_NO_QUANT = NoQuantConfig()
def quant_method_for(quant_config, layer, prefix: str):
"""The layer's Method from its config; a layer built without one is plain bf16."""
return (quant_config if quant_config is not None else _NO_QUANT).get_quant_method(layer, prefix)
_UNSET = object()
_QUANT_CONFIG = _UNSET
def set_quant_config(quant: QuantConfig | None) -> None:
"""Install the checkpoint's QuantConfig for the weight readers, which only get the model path; EngineConfig sets it once."""
global _QUANT_CONFIG
_QUANT_CONFIG = quant
def get_quant_config() -> QuantConfig | None:
if _QUANT_CONFIG is _UNSET:
raise RuntimeError("no QuantConfig installed: EngineConfig.model_config sets it before the weights load")
return _QUANT_CONFIG
__all__ = [
"QuantConfig", "quantization_config_of", "quant_method_for", "set_quant_config", "get_quant_config",
"NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config", "GGUFConfig",
"compressed_tensors", "fp8", "gguf", "modelopt", "mxfp4",
]