"""Explicit per-model AOT kernel-shape support table. The prebuilt kernel cache serves a runtime JIT lookup only on a byte-identical spec name, so the shape lists in :mod:`freetoken.kernel.aot` must cover every (model, weight-format) pair FreeToken serves. Mirroring flashinfer's ``aot`` module, the supported set is declared explicitly here -- one entry per checkpoint family, carrying the config.json fields its kernel shapes derive from -- and the aot spec lists are generated as the union over entries. Derivations, which must be kept in lockstep with the runtime call sites by hand (a drifted derivation misses the prebuilt cache by spec name and falls back to JIT, which needs nvcc): - store: ``element_size = num_kv_heads * head_dim * 2`` (bf16 KV row), one per paged-KV attention group (kvcache/mha_pool.py, kvcache/hybrid_swa_pool.py). DSV4 writes its MLA latent via torch scatter and contributes nothing. - index: ``element_size = hidden_size * 2`` (bf16 embedding row) paired with the runtime ``num_splits_for`` rule (layers/embedding.py -> kernel/index.py). DSV4 (plain nn.Embedding) and GGUF embeddings (ggml get_rows) bypass it. The whole table targets the shipped serving configuration: TP=1 (TP>1 shards kv heads, shrinking the store row) and a 2-byte compute dtype (``--dtype float32`` doubles the KV/embedding/bf16-bank rows). Both fall back to JIT. - fast_index_copy: per-expert row bytes of every offload bank the checkpoint's expert format registers (moe/offload_cache.py ``_BANK_SCHEMAS``); this is the per-bank fallback of ``copy_missing`` plus any caller that copies single banks. Geometry is TP=1 (the offload path's supported configuration). """ from __future__ import annotations from dataclasses import dataclass from .index import num_splits_for KV_CACHE_DTYPE_BYTES = 2 # every current model allocates bf16 paged KV EMBED_DTYPE_BYTES = 2 # embedding weights stay bf16 on the indexing() path @dataclass(frozen=True) class AotModel: """One supported checkpoint family and the config fields shapes need. ``kv_groups`` holds ``(num_kv_heads, head_dim)`` per paged-KV attention group (hybrid-SWA models have two); empty means the model bypasses the store kernel. ``expert_formats`` are ``_BANK_SCHEMAS`` keys this checkpoint is served with; empty for dense models. ``aliases`` are sibling checkpoints with identical shapes that this entry also covers. """ name: str architecture: str hidden_size: int kv_groups: tuple[tuple[int, int], ...] top_k: int | None = None moe_intermediate_size: int | None = None expert_formats: tuple[str, ...] = () embed_indexing: bool = True aliases: tuple[str, ...] = () # Further register.py keys this entry's checkpoints load under (text-only / # GGUF config variants of the same model); every registry key must be # claimed by some entry's architecture or arch_aliases. arch_aliases: tuple[str, ...] = () def fp8_block_scale_pad(rows: int, cols: int) -> int: """Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy).""" while (rows * cols * 2) % 16: cols += 1 return cols def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int) -> dict[str, int]: """Per-expert row bytes for each offload bank a format registers. Bank names and layouts follow moe/offload_cache.py ``_BANK_SCHEMAS`` and the per-format loaders cited inline; every value must stay a multiple of 16 for the fused multi-bank copy to engage. """ H, I = hidden_size, moe_intermediate_size if fmt == "bf16": # bf16 expert banks: gate_up [E, 2I, H], down [E, H, I] return {"gate_up": 2 * I * H * 2, "down": H * I * 2} if fmt == "fp8_block": # block-fp8 expert banks: fp8 weights + bf16 128x128 block # scales, trailing scale dim 16B-padded (same helper as the loader) B = 128 return { "gate_up": 2 * I * H, "gate_up_scale": (2 * I // B) * fp8_block_scale_pad(2 * I // B, H // B) * 2, "down": H * I, "down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2, } if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"): # models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp32 # per-row globals; marlin/b12x repacks are byte-identical with the globals # folded into GPU-resident alphas (layers/quantization/moe/nvfp4.py), so no global banks. banks = { "gate_up_packed": 2 * I * (H // 2), "gate_up_scale": 2 * I * (H // 16), "down_packed": H * (I // 2), "down_scale": H * (I // 16), } if fmt == "nvfp4": banks["gate_up_global"] = 2 * I * 4 banks["down_global"] = H * 4 return banks if fmt == "mxfp4_triton": # gpt-oss mxfp4 expert banks: transposed split-K blocks/scales + bf16 bias return { "gate_up_blocks": (H // 2) * (2 * I), "gate_up_scales": (H // 32) * (2 * I), "gate_up_bias": 2 * I * 2, "down_blocks": (I // 2) * H, "down_scales": (I // 32) * H, "down_bias": H * 2, } if fmt == "ds_fp4": # deepseek_v4/weight.py load_dsfp4_expert_sources: packed e2m1 + e8m0 per-32 scales return { "gate_up_packed": 2 * I * (H // 2), "gate_up_scale": 2 * I * (H // 32), "down_packed": H * (I // 2), "down_scale": H * (I // 32), } raise ValueError(f"Unknown expert bank format {fmt!r}") _NVFP4_FORMATS = ("nvfp4", "nvfp4_marlin", "nvfp4_b12x") # Config fields come from each checkpoint's config.json (text_config for the # multimodal wrappers); checkpoints without a local copy were recorded from # https://huggingface.co//raw/main/config.json. SUPPORTED_MODELS: tuple[AotModel, ...] = ( # ---- MoE checkpoints (offload expert banks) ---- AotModel( name="Qwen/Qwen3-30B-A3B", architecture="Qwen3MoeForCausalLM", hidden_size=2048, kv_groups=((4, 128),), top_k=8, moe_intermediate_size=768, expert_formats=("bf16",), aliases=("Qwen/Qwen3-30B-A3B-Thinking-2507",), # Qwen3-VL-30B-A3B: same text tower arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM", "Qwen3VLMoeGGUFForCausalLM"), ), AotModel( name="Qwen/Qwen3.5-35B-A3B", architecture="Qwen3_5MoeForConditionalGeneration", hidden_size=2048, kv_groups=((2, 256),), # full-attention group; GDN layers hold no paged KV top_k=8, moe_intermediate_size=512, expert_formats=("bf16",), arch_aliases=("Qwen3_5MoeForCausalLM", "Qwen3_5MoeGGUFForCausalLM"), # text-only release of the same tower ), AotModel( name="Qwen/Qwen3.5-35B-A3B-FP8", architecture="Qwen3_5MoeForConditionalGeneration", hidden_size=2048, kv_groups=((2, 256),), top_k=8, moe_intermediate_size=512, expert_formats=("fp8_block",), ), AotModel( name="Qwen/Qwen3.6-35B-A3B", architecture="Qwen3_5MoeForConditionalGeneration", hidden_size=2048, kv_groups=((2, 256),), top_k=8, moe_intermediate_size=512, expert_formats=("bf16",), ), AotModel( name="Qwen/Qwen3.6-35B-A3B-FP8", architecture="Qwen3_5MoeForConditionalGeneration", hidden_size=2048, kv_groups=((2, 256),), top_k=8, moe_intermediate_size=512, expert_formats=("fp8_block",), ), AotModel( name="nvidia/Qwen3.6-35B-A3B-NVFP4", architecture="Qwen3_5MoeForConditionalGeneration", hidden_size=2048, kv_groups=((2, 256),), top_k=8, moe_intermediate_size=512, expert_formats=_NVFP4_FORMATS, ), AotModel( # QSA compressed-sparse attention (12 of 48 layers): the QSAKVCache stores K/V # through store_cache (2 kv heads x 256 head_dim), the compressed index-key slab # and the pending ring write via the vendored qsa triton kernels. Hyper-connections # carry the residual, so the embedding row indexing() sees is still hidden_size. name="RadixArk/Qwen3.8-Flash-Next-NVFP4", architecture="Qwen4ExpForConditionalGeneration", hidden_size=2560, kv_groups=((2, 256),), top_k=10, moe_intermediate_size=640, expert_formats=(*_NVFP4_FORMATS, "fp8_block"), # GGUF expert rows differ per layer and reach the fused copy as runtime sizes arch_aliases=("Qwen4ExpGGUFForCausalLM",), ), AotModel( name="google/gemma-4-26B-A4B-it", architecture="Gemma4ForConditionalGeneration", hidden_size=2816, # full group: num_global_key_value_heads x global_head_dim; swa group: # num_key_value_heads x head_dim (gemma4/config.py attention_groups) kv_groups=((2, 512), (8, 256)), top_k=8, moe_intermediate_size=704, expert_formats=("bf16",), # GGUF expert rows differ per layer and reach the fused copy as runtime sizes arch_aliases=("Gemma4ForCausalLM", "Gemma4GGUFForCausalLM"), ), AotModel( name="nvidia/Gemma-4-26B-A4B-NVFP4", architecture="Gemma4ForConditionalGeneration", hidden_size=2816, kv_groups=((2, 512), (8, 256)), top_k=8, moe_intermediate_size=704, expert_formats=_NVFP4_FORMATS, ), AotModel( name="openai/gpt-oss-120b", architecture="GptOssForCausalLM", hidden_size=2880, kv_groups=((8, 64),), # sliding and full layers share the same geometry top_k=4, moe_intermediate_size=2880, expert_formats=("mxfp4_triton",), arch_aliases=("GptOssGGUFForCausalLM",), # the GGUF row builds the same classes ), AotModel( name="openai/gpt-oss-20b", architecture="GptOssForCausalLM", hidden_size=2880, kv_groups=((8, 64),), top_k=4, moe_intermediate_size=2880, expert_formats=("mxfp4_triton",), arch_aliases=("GptOssGGUFForCausalLM",), ), AotModel( name="zai-org/GLM-4.7", architecture="Glm4MoeForCausalLM", hidden_size=5120, kv_groups=((8, 128),), top_k=8, moe_intermediate_size=1536, expert_formats=("bf16",), arch_aliases=("Glm4MoeGGUFForCausalLM",), ), AotModel( name="nvidia/GLM-4.7-NVFP4", architecture="Glm4MoeForCausalLM", hidden_size=5120, kv_groups=((8, 128),), top_k=8, moe_intermediate_size=1536, expert_formats=_NVFP4_FORMATS, ), AotModel( # MLA latent-KV: the dsa backend writes the latent via torch scatter, # not store_cache (same as DSV4 below), so no paged-KV store groups. name="zai-org/GLM-5.2-NVFP4", architecture="GlmMoeDsaForCausalLM", hidden_size=6144, kv_groups=(), top_k=8, moe_intermediate_size=2048, expert_formats=_NVFP4_FORMATS, ), AotModel( # GLM-5.3-Flash: hybrid KDA + NoPE-MLA/DSA (kpool indexer). Latent writes # go through torch scatter like GLM-5.2 (no paged-KV store groups); the # KDA conv/recurrent state lives in the LinearStatePool, not paged KV. name="RedHatAI/GLM-5.3-Flash-NVFP4", architecture="Glm5NextForCausalLM", # GGUF expert rows differ per layer and reach the fused copy as runtime sizes arch_aliases=("Glm5NextForConditionalGeneration", "Glm5NextGGUFForCausalLM"), hidden_size=4096, kv_groups=(), top_k=8, moe_intermediate_size=2048, expert_formats=(*_NVFP4_FORMATS, "fp8_block"), aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"), ), AotModel( # MiniMaxAI/MiniMax-M2.5 ships block-fp8, which has no expert-bank # provider for this arch on main -- the NVFP4 release is the servable # offload path, and both share the same attention/embedding shapes. name="nvidia/MiniMax-M2.5-NVFP4", architecture="MiniMaxM2ForCausalLM", hidden_size=3072, kv_groups=((8, 128),), top_k=8, moe_intermediate_size=1536, # experts reuse the dense intermediate_size expert_formats=_NVFP4_FORMATS, aliases=("MiniMaxAI/MiniMax-M2.5",), arch_aliases=("MiniMaxM2GGUFForCausalLM",), ), AotModel( # GQA block-sparse (BSA): the BSAKVCache stores K/V through store_cache # (4 kv heads x 128 head_dim), the index-key slab writes via torch scatter. # Routed experts are NVFP4 with swigluoai, which restricts the expert GEMM # to the Triton kernels -- but the banks keep the native "nvfp4" layout, # so its bank rows are what fast_index_copy sees. # The upstream MiniMaxAI/MiniMax-M3 alias covers the attention/embedding # shapes only: its experts are MXFP8, which has no expert-bank provider on # main -- the NVFP4 release is the servable offload path (M2.5 precedent). name="nvidia/MiniMax-M3-NVFP4", architecture="MiniMaxM3SparseForConditionalGeneration", hidden_size=6144, kv_groups=((4, 128),), top_k=4, moe_intermediate_size=3072, expert_formats=("nvfp4",), aliases=("MiniMaxAI/MiniMax-M3",), arch_aliases=("MiniMaxM3SparseForCausalLM", "MiniMaxM3GGUFForCausalLM"), ), AotModel( name="deepseek-ai/DeepSeek-V4-Flash", architecture="DeepseekV4ForCausalLM", # llama.cpp deepseek4 GGUF (unsloth 0731 GGUF; text only) arch_aliases=("DeepseekV4GGUFForCausalLM",), hidden_size=4096, kv_groups=(), # MLA latent pool writes via torch scatter, not store_cache top_k=6, moe_intermediate_size=2048, expert_formats=("ds_fp4",), embed_indexing=False, # plain nn.Embedding ), AotModel( # DSV41 pools are packed byte rows written by the dsv41 pack kernel, not store_cache; # the routed experts share DSV4's ds_fp4 bank layout (hidden 5120, intermediate 2304). name="deepseek-ai/DeepSeek-V4.1-Flash", architecture="DeepseekV41ForCausalLM", hidden_size=5120, kv_groups=(), top_k=6, moe_intermediate_size=2304, expert_formats=("ds_fp4",), ), AotModel( name="RecursiveSI/flint", architecture="FlintForCausalLM", hidden_size=2048, kv_groups=((4, 128),), top_k=6, moe_intermediate_size=512, expert_formats=("bf16", *_NVFP4_FORMATS), aliases=("FutureMLS-Lab/flint-nvfp4-coldgptq",), arch_aliases=("FlintGGUFForCausalLM",), ), # ---- dense checkpoints (store/index only, no expert banks) ---- AotModel( name="Qwen/Qwen3.6-27B", architecture="Qwen3_5ForConditionalGeneration", hidden_size=5120, kv_groups=((4, 256),), aliases=("Qwen/Qwen3.6-27B-FP8", "nvidia/Qwen3.6-27B-NVFP4"), arch_aliases=("Qwen3_5ForCausalLM", "Qwen3_5GGUFForCausalLM"), # text-only release of the same tower ), AotModel( name="google/gemma-4-12B-it", architecture="Gemma4UnifiedForConditionalGeneration", hidden_size=3840, kv_groups=((1, 512), (8, 256)), arch_aliases=("Gemma4UnifiedForCausalLM", "Gemma4UnifiedGGUFForCausalLM"), ), AotModel( name="nvidia/Gemma-4-31B-IT-NVFP4", architecture="Gemma4ForConditionalGeneration", hidden_size=5376, kv_groups=((4, 512), (16, 256)), ), AotModel( name="meta-models/Muse-Glimmer-30B", architecture="MuseGlimmerForConditionalGeneration", hidden_size=6656, kv_groups=((2, 128),), # sliding and full (NoPE) layers share the same geometry aliases=("RedHatAI/Muse-Glimmer-30B-NVFP4",), arch_aliases=("MuseGlimmerGGUFForCausalLM",), ), AotModel( name="meta-llama/Llama-3.1-8B-Instruct", architecture="LlamaForCausalLM", hidden_size=4096, kv_groups=((8, 128),), arch_aliases=("LlamaGGUFForCausalLM",), ), AotModel( name="Qwen/Qwen2-7B", architecture="Qwen2ForCausalLM", hidden_size=3584, kv_groups=((4, 128),), arch_aliases=("Qwen2GGUFForCausalLM",), ), AotModel( name="Qwen/Qwen3-8B", architecture="Qwen3ForCausalLM", hidden_size=4096, kv_groups=((8, 128),), # Qwen3-VL: same text tower arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM", "Qwen3VLGGUFForCausalLM"), ), AotModel( name="mistralai/Mistral-7B-Instruct-v0.3", architecture="MistralForCausalLM", hidden_size=4096, kv_groups=((8, 128),), ), AotModel( name="mistralai/Mistral-Small-3.1-24B-Instruct-2503", architecture="Mistral3ForConditionalGeneration", hidden_size=5120, kv_groups=((8, 128),), ), ) def store_element_sizes(model: AotModel) -> set[int]: return {kv * hd * KV_CACHE_DTYPE_BYTES for kv, hd in model.kv_groups} def index_variants(model: AotModel) -> set[tuple[int, int]]: if not model.embed_indexing: return set() element_size = model.hidden_size * EMBED_DTYPE_BYTES return {(element_size, num_splits_for(element_size))} def fast_index_copy_feature_sizes(model: AotModel) -> set[int]: sizes: set[int] = set() for fmt in model.expert_formats: assert model.moe_intermediate_size is not None sizes.update( expert_bank_row_bytes(fmt, model.hidden_size, model.moe_intermediate_size).values() ) return sizes def aggregate_store_element_sizes() -> tuple[int, ...]: sizes: set[int] = set() for model in SUPPORTED_MODELS: sizes.update(store_element_sizes(model)) return tuple(sorted(sizes)) def aggregate_index_variants() -> tuple[tuple[int, int], ...]: variants: set[tuple[int, int]] = set() for model in SUPPORTED_MODELS: variants.update(index_variants(model)) return tuple(sorted(variants)) # Synthetic per-bank sizes tests/moe/test_fused_copy.py exercises (its FEATS list); # prebuilt so the suite also runs under FREETOKEN_DISABLE_JIT=1 release validation. TEST_FEATURE_SIZES = (512, 8192) def aggregate_fast_index_copy_feature_sizes() -> tuple[int, ...]: sizes: set[int] = set(TEST_FEATURE_SIZES) for model in SUPPORTED_MODELS: sizes.update(fast_index_copy_feature_sizes(model)) # the per-bank kernel copies rows in fixed 128-byte steps; other sizes cannot compile return tuple(sorted(size for size in sizes if size % 128 == 0)) __all__ = [ "AotModel", "SUPPORTED_MODELS", "TEST_FEATURE_SIZES", "aggregate_fast_index_copy_feature_sizes", "aggregate_index_variants", "aggregate_store_element_sizes", "expert_bank_row_bytes", "fast_index_copy_feature_sizes", "index_variants", "store_element_sizes", ]