Files
Xiaoze Fanandzihaomu e5d3bd3faf feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
* build(kernel): vendor ggml-cuda from llama.cpp 8c2134a8 behind a tvm-ffi binding

* build(deps): vendor the gguf-py reader

* feat(quant): add the GGUF quant kind and an embedding quant method

* feat(gguf): serve Flash-Next and Gemma4 GGUF on the GGUF quant kind and drop the old Q4_0 path

* fix(kernel): build ggml-cuda for plain sm_120 by turning off the native fp4 MMA

* perf(gguf): write ggml results in the activation dtype and keep MoE intermediates in bf16

* build(kernel): compress the ggml fatbin to keep the multi-arch kernel cache small

* fix(gguf): read F32, F16 and BF16 PLE tables without ggml

* fix(moe): pad the offload slot cache for ggml MMQ tail reads

* fix(gguf): support hip builds and runtime devices

* fix(gguf): copy same-type dequantize and check for missing ggml converters

* fix(gguf): load quantized qwen4exp ple_key and ple_value

* feat(gguf): run float parts of a fused GGUF linear as bf16 segments

* fix(gguf): load quantized qwen4exp hyper-connection and gdn ba weights

* fix(moe): zero the prefill staging area after the last prefill layer

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-15 16:33:16 -07:00

493 lines
19 KiB
Python

import os
from typing import TYPE_CHECKING, Iterator, Tuple
import torch
from freetoken.core import get_global_ctx
from freetoken.distributed import DistributedCommunicator, get_tp_info
from freetoken.moe import is_offload_moe_strategy
from freetoken.moe.fused import fused_topk
from freetoken.moe.offload_cache import OffloadMoeCache
from .base import BaseOP
from .quantization import ExpertView, LayerKind, QuantConfig, quant_method_for
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
# Router decision (topk_weights[float32], topk_ids[int32]) for models whose router
# is computed outside the MoE layer. Such models call ``routed_forward`` with a
# precomputed routing instead of going through the generic softmax+top-k path.
TopK = Tuple[torch.Tensor, torch.Tensor]
# Hybrid decode overlaps the CPU overflow GEMV behind the GPU PCIe fetch + GEMM by
# default. Set FREETOKEN_HYBRID_OVERLAP=0 to force the serial path (CPU sync before the
# GPU work) -- a measurement-only escape hatch to A/B the overlap benefit.
_HYBRID_OVERLAP = os.getenv("FREETOKEN_HYBRID_OVERLAP", "1") != "0"
class MoELayer(BaseOP):
"""Resident routed experts.
The expert format comes from ``quant_method`` (declared by ``create_weights``, run by
``apply``); without a ``quant_config`` the experts are plain bf16. The gated activation is
``act(clamp(g, limit) * alpha) * (clamp(u) + beta)`` with ``interleaved`` gate|up rows
for gpt-oss."""
quant_layer_kind = LayerKind.MOE
def __init__(
self,
num_experts: int,
top_k: int,
hidden_size: int,
intermediate_size: int,
renormalize: bool = True,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
allocate_experts: bool = True,
*,
alpha: float = 1.0,
beta: float = 0.0,
limit: float | None = None,
interleaved: bool = False,
has_bias: bool = False,
layer_id: int | None = None,
strategy: str = "resident",
decode_target: str = "gpu",
quant_config: QuantConfig | None = None,
prefix: str = "",
):
super().__init__()
self.num_experts = num_experts
self.top_k = top_k
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self._comm = DistributedCommunicator()
tp_info = get_tp_info()
self.tp_rank = tp_info.rank
self.tp_size = tp_size = tp_info.size
self.renormalize = renormalize
self.activation = activation
self.apply_router_weight_on_input = apply_router_weight_on_input
self.alpha = alpha
self.beta = beta
self.limit = limit
self.interleaved = interleaved
self.has_bias = has_bias
self.layer_id = layer_id
self.strategy = strategy
self.decode_target = decode_target
self.prefix = prefix
self.quant_method = quant_method_for(quant_config, self, prefix)
if allocate_experts:
self.quant_method.create_weights(self)
def finalize(self) -> None:
self.quant_method.finalize(self)
def _maybe_all_reduce(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self.tp_size > 1:
return self._comm.all_reduce(hidden_states)
return hidden_states
def _resident_gemm(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
return self.quant_method.apply(
hidden_states, topk_weights, topk_ids, self.quant_method.resident_view(self),
layer=self, is_prefill=get_global_ctx().batch.is_prefill,
)
def routed_forward(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Expert compute for an externally computed routing decision (``TopK``).
Same name and shape as ``OffloadMoELayer.routed_forward`` so a model with
its own router calls ``experts.routed_forward(...)`` without knowing whether
the experts are resident or offloaded. The shared contract is the offload
one: ``topk_ids`` must be safe to mutate in place (the offload decode
rewrites expert ids into cache slot ids); pass a fresh tensor or a clone.
The resident path does not mutate it today, but callers must not rely on
that.
``hidden_states`` may also be overwritten by the expert kernel. Compute
shared branches that need the original input before calling this method.
"""
out = self._resident_gemm(hidden_states, topk_weights, topk_ids)
return self._maybe_all_reduce(out)
def forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor | None = None,
):
topk_weights, topk_ids = fused_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=self.top_k,
renormalize=self.renormalize,
)
return self._maybe_all_reduce(self._resident_gemm(hidden_states, topk_weights, topk_ids))
class OffloadMoELayer(MoELayer):
def __init__(
self,
layer_id: int,
num_experts: int,
top_k: int,
hidden_size: int,
intermediate_size: int,
renormalize: bool = True,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
*,
alpha: float = 1.0,
beta: float = 0.0,
limit: float | None = None,
interleaved: bool = False,
has_bias: bool = False,
strategy: str = "offload",
decode_target: str = "gpu",
quant_config: QuantConfig | None = None,
prefix: str = "",
):
super().__init__(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
renormalize=renormalize,
activation=activation,
apply_router_weight_on_input=apply_router_weight_on_input,
allocate_experts=False,
alpha=alpha,
beta=beta,
limit=limit,
interleaved=interleaved,
has_bias=has_bias,
layer_id=layer_id,
strategy=strategy,
decode_target=decode_target,
quant_config=quant_config,
prefix=prefix,
)
self.offload_cache: OffloadMoeCache | None = None
@property
def skips_inactive_slots(self) -> bool:
return self.quant_method.kernel.supports_inactive_slots
def forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor | None = None,
):
ctx = get_global_ctx()
if ctx.batch.is_prefill:
final_hidden_states = self.prefill_forward(hidden_states, router_logits)
else:
final_hidden_states = self.decode_forward(hidden_states, router_logits)
return self._maybe_all_reduce(final_hidden_states)
def routed_forward(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Expert compute for an externally computed routing decision (``TopK``).
The entry point for models whose router does not fit ``fused_topk`` (sigmoid
scores, selection bias, group-limited top-k, ...); identical to ``forward``
past the router. ``topk_ids`` must be safe to mutate in place (decode
rewrites expert ids into cache slot ids); pass a fresh tensor or a clone.
``hidden_states`` may also be overwritten by the expert kernel. Compute
shared branches that need the original input before calling this method.
"""
ctx = get_global_ctx()
if ctx.batch.is_prefill:
out = self._prefill_routed(hidden_states, topk_weights, topk_ids)
else:
out = self._decode_routed(hidden_states, topk_weights, topk_ids)
return self._maybe_all_reduce(out)
def decode_forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor | None = None,
):
topk_weights, topk_ids = fused_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=self.top_k,
renormalize=self.renormalize,
)
return self._decode_routed(hidden_states, topk_weights, topk_ids)
def prefill_forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor | None = None,
):
topk_weights, topk_ids = fused_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=self.top_k,
renormalize=self.renormalize,
)
return self._prefill_routed(hidden_states, topk_weights, topk_ids)
# ------------------------------------------------------------------
# Data movement -- one decision tree for every quant format (the banks
# registry makes the cache machinery bank-count agnostic). Decode loads
# on demand; prefill streams whole layers, double-buffered when overlap
# is enabled. The kernels only ever see bank views plus row indices;
# which kernel runs is decided afterwards, in ``_expert_gemm``.
# ------------------------------------------------------------------
def _decode_routed(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""On-demand load: ``ensure_experts`` rewrites ``topk_ids`` into cache slot
ids in place (loading missing experts), then the GEMM reads the full slot
cache. All device-side with fixed shapes, so the decode call is CUDA-graph
capturable.
For ``decode_target == "cpu"`` the experts are instead computed on the CPU
(high RAM bandwidth) straight from the host banks: ship hidden/routing to
pinned host memory, run the GEMV on the worker pool via host nodes, ship the
result back. The GPU slot cache is untouched (topk_ids keep their raw expert
ids), so no ``ensure_experts``/``copy_missing`` here."""
cache = self.offload_cache
assert cache is not None
if cache.is_cpu_layer(self.layer_id):
executor = cache.cpu_executor
assert executor is not None, "CPU MoE executor was not initialized"
return executor.decode(self.layer_id, hidden_states, topk_weights, topk_ids)
if cache.decode_target == "hybrid":
return self._decode_hybrid(cache, hidden_states, topk_weights, topk_ids)
cache.ensure_experts(self.layer_id, topk_ids)
cache.copy_missing()
return self._expert_gemm(
cache,
hidden_states,
topk_weights,
topk_ids,
views=cache.pool_bank_views(self.layer_id),
n=None,
alphas=cache.alphas_for_slots(self.layer_id),
is_prefill=False,
)
def _decode_hybrid(
self,
cache: OffloadMoeCache,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Hybrid decode: GPU computes cache hits + <=K freshly-fetched experts, the CPU
computes the overflow misses, overlapped, then the partials merge.
The CPU pool is kicked off (``decode_submit``) before the GPU PCIe fetch + GEMM so
the CPU overflow GEMV runs concurrently with the GPU work. Capture-safe: the
routing split runs inside the hybrid ensure kernel and the CPU submit/sync are host nodes.
Each route is computed exactly once -- CPU-assigned routes reach the GPU as slot -1 with
weight 0 (a kernel with ``supports_inactive_slots`` skips them without a read; any other reads
slot 0, whose storage is a payload or the bank's zero fill) and the CPU ids are -1 for
GPU-assigned routes (the C++ kernel skips id<0).
"""
executor = cache.cpu_executor
assert executor is not None, "CPU MoE executor was not initialized"
# the kernel reads every route's slot: give the inactive ones slot 0, which holds a
# complete payload or the bank's zero fill (never uninitialized storage), times weight 0
# with several quants this holds only because the cache zeroes its staging area after prefill
clamp = not self.skips_inactive_slots
gpu_slots, gpu_w, cpu_ids = cache.ensure_experts_hybrid(self.layer_id, topk_ids, topk_weights, clamp=clamp)
pending = executor.decode_submit(self.layer_id, hidden_states, topk_weights, cpu_ids)
# Measurement knob: FREETOKEN_HYBRID_OVERLAP=0 syncs the CPU pool *before* the
# PCIe fetch + GPU GEMM, serializing the two so an A/B isolates the overlap win.
cpu_routed_early = (
executor.decode_sync(pending) if not _HYBRID_OVERLAP else None
)
cache.copy_missing()
gpu_routed = self._expert_gemm(
cache,
hidden_states,
gpu_w,
gpu_slots,
views=cache.pool_bank_views(self.layer_id),
n=None,
alphas=cache.alphas_for_slots(self.layer_id),
is_prefill=False,
)
if not _HYBRID_OVERLAP:
return gpu_routed + cpu_routed_early
return executor.decode_sync(pending, add_to=gpu_routed)
def _prefill_routed(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Prefill movement: stream whole layers -- double-buffered behind the
previous layer's GEMMs when ``prefill_overlap`` is on, else a synchronous
``materialize_layer``. In both, position == expert id, so the routing ids
pass through unmapped."""
cache = self.offload_cache
assert cache is not None
if cache.prefill_overlap:
views = self._wait_prefill_overlap(cache)
out = self._expert_gemm(
cache,
hidden_states,
topk_weights,
topk_ids,
views=views,
n=self.num_experts,
alphas=cache.alphas_for_layer(self.layer_id),
is_prefill=True,
)
cache.release_prefill_layer(self.layer_id)
return out
cache.materialize_layer(self.layer_id)
cache.copy_missing()
out = self._expert_gemm(
cache,
hidden_states,
topk_weights,
topk_ids,
views=cache.stage_views(self.layer_id, 0),
n=self.num_experts,
alphas=cache.alphas_for_layer(self.layer_id),
is_prefill=True,
)
cache.release_prefill_layer(self.layer_id)
return out
def _wait_prefill_overlap(self, cache: OffloadMoeCache) -> tuple[torch.Tensor, ...]:
"""Double-buffer choreography for this layer's overlap prefill: kick off the
next layer's full-layer H2D copy, then return this layer's bank views (in
bank registration order; buffer position == expert id, so routing ids pass
through unmapped). The caller runs ``release_prefill_layer`` after its GEMMs.
"""
if self.layer_id == 0:
cache.begin_prefill()
cache.prefetch_prefill_layer(self.layer_id)
cache.prefetch_prefill_layer(self.layer_id + 1)
return cache.wait_prefill_layer(self.layer_id)
# ------------------------------------------------------------------
# Kernel dispatch: ``views`` are the bank tensors the movement step produced (in bank registration order) and ``topk_ids`` already index their rows.
# ------------------------------------------------------------------
def _expert_gemm(
self,
cache: OffloadMoeCache,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
*,
views: tuple[torch.Tensor, ...],
n: int | None,
alphas: tuple[torch.Tensor, torch.Tensor] | None,
is_prefill: bool,
) -> torch.Tensor:
from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package
view = ExpertView(
{canonical_role(name): t for name, t in zip(cache.bank_schema, views)},
slots=None if n is not None else topk_ids, n=n, alphas=alphas,
)
return self.quant_method.apply(
hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill
)
def iter_moe_layers(model) -> Iterator[MoELayer]:
if isinstance(model, MoELayer):
yield model
if not isinstance(model, BaseOP):
return
for value in model.__dict__.values():
if isinstance(value, BaseOP):
yield from iter_moe_layers(value)
elif isinstance(value, (list, tuple)):
for item in value:
yield from iter_moe_layers(item)
def make_moe_layer(
config: "ModelConfig",
*,
layer_id: int | None = None,
activation: str = "silu",
renormalize: bool | None = None,
apply_router_weight_on_input: bool = False,
num_experts: int | None = None,
top_k: int | None = None,
hidden_size: int | None = None,
intermediate_size: int | None = None,
resident_cls: type[MoELayer] | None = None,
offload_cls: "type[OffloadMoELayer] | None" = None,
alpha: float = 1.0,
beta: float = 0.0,
limit: float | None = None,
interleaved: bool = False,
has_bias: bool = False,
quant_config: QuantConfig | None = None,
prefix: str = "",
) -> MoELayer:
"""Build the experts layer for ``config.moe_strategy`` -- the one construction
seam between a model and the MoE strategy.
Picks ``OffloadMoELayer`` for the offload family (offload/cpu/hybrid) and
``MoELayer`` otherwise. Geometry defaults come from ``config``; pass overrides
for models whose fields deviate. ``resident_cls``/``offload_cls`` keep model-specific
subclasses constructible through the same seam.
"""
offload = is_offload_moe_strategy(config.moe_strategy)
layer_cls = (offload_cls or OffloadMoELayer) if offload else (resident_cls or MoELayer)
kwargs = dict(
num_experts=num_experts if num_experts is not None else config.num_experts,
top_k=top_k if top_k is not None else config.num_experts_per_tok,
hidden_size=hidden_size if hidden_size is not None else config.hidden_size,
intermediate_size=(
intermediate_size if intermediate_size is not None else config.moe_intermediate_size
),
renormalize=renormalize if renormalize is not None else config.norm_topk_prob,
activation=activation,
apply_router_weight_on_input=apply_router_weight_on_input,
alpha=alpha,
beta=beta,
limit=limit,
interleaved=interleaved,
has_bias=has_bias,
quant_config=quant_config,
prefix=prefix,
layer_id=layer_id,
)
if offload:
assert layer_id is not None, "offload MoE backends need the layer_id"
kwargs["strategy"] = config.moe_strategy
kwargs["decode_target"] = config.decode_target
return layer_cls(**kwargs)