mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding * build(deps): vendor the gguf-py reader * feat(quant): add the GGUF quant kind and an embedding quant method * feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path * fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA * perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16 * build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small * fix(gguf): read F32, F16 and BF16 PLE tables without ggml * fix(moe): pad the offload slot cache for ggml MMQ tail reads * fix(gguf): support hip builds and runtime devices * fix(gguf): copy same-type dequantize and check for missing ggml converters * fix(gguf): load quantized qwen4exp ple_key and ple_value * feat(gguf): run float parts of a fused GGUF linear as bf16 segments * fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights * fix(moe): zero the prefill staging area after the last prefill layer --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
56 lines
2.1 KiB
Python
56 lines
2.1 KiB
Python
from .scheme import GroupShape, QuantKind, QuantScheme, WeightDesc
|
|
from .registry import LayerKind, dialects, method_class, methods_for, register_dialect, register_method
|
|
from .method import KernelSelectionError, QuantMethod, finalize_quant, select_kernel
|
|
from .embedding import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod, GGUFEmbeddingMethod, UnquantizedEmbeddingMethod
|
|
from .linear import (
|
|
Fp8BlockLinearMethod,
|
|
Fp8TensorLinearMethod,
|
|
LinearConfig,
|
|
LinearKernel,
|
|
LinearMethod,
|
|
Mxfp8LinearMethod,
|
|
Nvfp4LinearMethod,
|
|
UnquantizedLinearMethod,
|
|
)
|
|
from .moe import (
|
|
BankSpec,
|
|
ExpertView,
|
|
Fp8BlockMoEMethod,
|
|
MoEConfig,
|
|
MoEKernel,
|
|
MoEMethod,
|
|
Mxfp4MoEMethod,
|
|
Mxfp8MoEMethod,
|
|
Nvfp4MoEMethod,
|
|
UnquantizedMoEMethod,
|
|
)
|
|
from .quant_backend import QuantBackend, get_quant_backend, set_quant_backend
|
|
from .names import NameMap
|
|
from .configs import (
|
|
CompressedTensorsConfig,
|
|
Fp8BlockConfig,
|
|
GGUFConfig,
|
|
ModelOptConfig,
|
|
Mxfp4Config,
|
|
NoQuantConfig,
|
|
QuantConfig,
|
|
get_quant_config,
|
|
quant_method_for,
|
|
quantization_config_of,
|
|
set_quant_config,
|
|
)
|
|
|
|
__all__ = [
|
|
"QuantKind", "GroupShape", "QuantScheme", "WeightDesc",
|
|
"LayerKind", "dialects", "method_class", "methods_for", "register_dialect", "register_method",
|
|
"KernelSelectionError", "select_kernel", "QuantMethod",
|
|
"EmbeddingConfig", "EmbeddingKernel", "EmbeddingMethod", "UnquantizedEmbeddingMethod", "GGUFEmbeddingMethod",
|
|
"LinearConfig", "LinearKernel", "LinearMethod",
|
|
"UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",
|
|
"BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",
|
|
"UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",
|
|
"QuantBackend", "set_quant_backend", "get_quant_backend", "NameMap",
|
|
"QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "GGUFConfig", "Mxfp4Config",
|
|
"quant_method_for", "quantization_config_of", "set_quant_config", "get_quant_config", "finalize_quant",
|
|
]
|