mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor(quant): config, scheme and method layers for checkpoint quantization QuantConfig parses the checkpoint's quantization_config into a QuantScheme per projection; every Linear*, ParallelLMHead and MoE layer takes quant_config and a prefix, declares its tensors through a QuantMethod and selects its kernel at construction. MoE kernels own the expert bank layout, pack and apply, so the offload cache, the FTW converter and the CPU executor follow one layout. - --quant-backend layer[.kind]=name selects kernels (--nvfp4-backend forwards to it) - --moe-strategy replaces --moe-backend; the old spelling stays as an alias - --moe-cpu-layers auto for Windows/WSL pin budgets; --moe-cache-auto respects the kernel's slot limit - removes the per-family quantized layer classes, bank providers, format env switches and GLM runtime fp8
18 lines
880 B
Python
18 lines
880 B
Python
from __future__ import annotations
|
|
|
|
# Where routed experts live. The offload family (offload / cpu / hybrid) serves experts from
|
|
# pinned host banks through an ``OffloadMoeCache`` -- the GPU only holds the two-layer prefill
|
|
# double buffer -- and differs only in how decode gets the experts: ``offload`` streams the
|
|
# missing experts over PCIe into a GPU slot cache, ``cpu`` computes them on the CPU from the
|
|
# host banks, ``hybrid`` fetches at most K missing experts per layer and computes the rest on
|
|
# the CPU, overlapped. ``fused`` keeps every expert resident on the GPU.
|
|
MOE_STRATEGIES = ("fused", "offload", "cpu", "hybrid")
|
|
OFFLOAD_MOE_STRATEGIES = frozenset({"offload", "cpu", "hybrid"})
|
|
|
|
|
|
def is_offload_moe_strategy(strategy: str) -> bool:
|
|
return strategy in OFFLOAD_MOE_STRATEGIES
|
|
|
|
|
|
__all__ = ["MOE_STRATEGIES", "OFFLOAD_MOE_STRATEGIES", "is_offload_moe_strategy"]
|