mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(ricursive): add ripple-65b preview support * fix(ricursive): read swiglu_clamp as a per-layer gguf array llama.cpp's Ripple converter (ggml-org/llama.cpp#31692) writes ricursive.swiglu_clamp with one value per layer, while the first community GGUFs store a scalar. Accept both and refuse per-layer values that differ, since the model has a single clamp. * fix(ricursive): add ripple's own parsers and copy the dense reader
499 lines
19 KiB
Python
499 lines
19 KiB
Python
"""Explicit per-model AOT kernel-shape support table.
|
|
|
|
The prebuilt kernel cache serves a runtime JIT lookup only on a byte-identical
|
|
spec name, so the shape lists in :mod:`freetoken.kernel.aot` must cover every
|
|
(model, weight-format) pair FreeToken serves. Mirroring flashinfer's ``aot``
|
|
module, the supported set is declared explicitly here -- one entry per
|
|
checkpoint family, carrying the config.json fields its kernel shapes derive
|
|
from -- and the aot spec lists are generated as the union over entries.
|
|
|
|
Derivations, which must be kept in lockstep with the runtime call sites by hand
|
|
(a drifted derivation misses the prebuilt cache by spec name and falls back to
|
|
JIT, which needs nvcc):
|
|
|
|
- store: ``element_size = num_kv_heads * head_dim * 2`` (bf16 KV row), one per
|
|
paged-KV attention group (kvcache/mha_pool.py, kvcache/hybrid_swa_pool.py).
|
|
DSV4 writes its MLA latent via torch scatter and contributes nothing.
|
|
- index: ``element_size = hidden_size * 2`` (bf16 embedding row) paired with
|
|
the runtime ``num_splits_for`` rule (layers/embedding.py -> kernel/index.py).
|
|
DSV4 (plain nn.Embedding) and GGUF embeddings (ggml get_rows) bypass it.
|
|
|
|
The whole table targets the shipped serving configuration: TP=1 (TP>1 shards
|
|
kv heads, shrinking the store row) and a 2-byte compute dtype (``--dtype
|
|
float32`` doubles the KV/embedding/bf16-bank rows). Both fall back to JIT.
|
|
- fast_index_copy: per-expert row bytes of every offload bank the checkpoint's
|
|
expert format registers (moe/offload_cache.py ``_BANK_SCHEMAS``); this is the
|
|
per-bank fallback of ``copy_missing`` plus any caller that copies single
|
|
banks. Geometry is TP=1 (the offload path's supported configuration).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from .index import num_splits_for
|
|
|
|
KV_CACHE_DTYPE_BYTES = 2 # every current model allocates bf16 paged KV
|
|
EMBED_DTYPE_BYTES = 2 # embedding weights stay bf16 on the indexing() path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AotModel:
|
|
"""One supported checkpoint family and the config fields shapes need.
|
|
|
|
``kv_groups`` holds ``(num_kv_heads, head_dim)`` per paged-KV attention
|
|
group (hybrid-SWA models have two); empty means the model bypasses the
|
|
store kernel. ``expert_formats`` are ``_BANK_SCHEMAS`` keys this checkpoint
|
|
is served with; empty for dense models. ``aliases`` are sibling checkpoints
|
|
with identical shapes that this entry also covers.
|
|
"""
|
|
|
|
name: str
|
|
architecture: str
|
|
hidden_size: int
|
|
kv_groups: tuple[tuple[int, int], ...]
|
|
top_k: int | None = None
|
|
moe_intermediate_size: int | None = None
|
|
expert_formats: tuple[str, ...] = ()
|
|
embed_indexing: bool = True
|
|
aliases: tuple[str, ...] = ()
|
|
# Further register.py keys this entry's checkpoints load under (text-only /
|
|
# GGUF config variants of the same model); every registry key must be
|
|
# claimed by some entry's architecture or arch_aliases.
|
|
arch_aliases: tuple[str, ...] = ()
|
|
|
|
|
|
def fp8_block_scale_pad(rows: int, cols: int) -> int:
|
|
"""Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy)."""
|
|
while (rows * cols * 2) % 16:
|
|
cols += 1
|
|
return cols
|
|
|
|
|
|
def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int) -> dict[str, int]:
|
|
"""Per-expert row bytes for each offload bank a format registers.
|
|
|
|
Bank names and layouts follow moe/offload_cache.py ``_BANK_SCHEMAS`` and the
|
|
per-format loaders cited inline; every value must stay a multiple of 16 for
|
|
the fused multi-bank copy to engage.
|
|
"""
|
|
H, I = hidden_size, moe_intermediate_size
|
|
if fmt == "bf16":
|
|
# bf16 expert banks: gate_up [E, 2I, H], down [E, H, I]
|
|
return {"gate_up": 2 * I * H * 2, "down": H * I * 2}
|
|
if fmt == "fp8_block":
|
|
# block-fp8 expert banks: fp8 weights + bf16 128x128 block
|
|
# scales, trailing scale dim 16B-padded (same helper as the loader)
|
|
B = 128
|
|
return {
|
|
"gate_up": 2 * I * H,
|
|
"gate_up_scale": (2 * I // B) * fp8_block_scale_pad(2 * I // B, H // B) * 2,
|
|
"down": H * I,
|
|
"down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,
|
|
}
|
|
if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):
|
|
# models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp32
|
|
# per-row globals; marlin/b12x repacks are byte-identical with the globals
|
|
# folded into GPU-resident alphas (layers/quantization/moe/nvfp4.py), so no global banks.
|
|
banks = {
|
|
"gate_up_packed": 2 * I * (H // 2),
|
|
"gate_up_scale": 2 * I * (H // 16),
|
|
"down_packed": H * (I // 2),
|
|
"down_scale": H * (I // 16),
|
|
}
|
|
if fmt == "nvfp4":
|
|
banks["gate_up_global"] = 2 * I * 4
|
|
banks["down_global"] = H * 4
|
|
return banks
|
|
if fmt == "mxfp4_triton":
|
|
# gpt-oss mxfp4 expert banks: transposed split-K blocks/scales + bf16 bias
|
|
return {
|
|
"gate_up_blocks": (H // 2) * (2 * I),
|
|
"gate_up_scales": (H // 32) * (2 * I),
|
|
"gate_up_bias": 2 * I * 2,
|
|
"down_blocks": (I // 2) * H,
|
|
"down_scales": (I // 32) * H,
|
|
"down_bias": H * 2,
|
|
}
|
|
if fmt == "ds_fp4":
|
|
# deepseek_v4/weight.py load_dsfp4_expert_sources: packed e2m1 + e8m0 per-32 scales
|
|
return {
|
|
"gate_up_packed": 2 * I * (H // 2),
|
|
"gate_up_scale": 2 * I * (H // 32),
|
|
"down_packed": H * (I // 2),
|
|
"down_scale": H * (I // 32),
|
|
}
|
|
raise ValueError(f"Unknown expert bank format {fmt!r}")
|
|
|
|
|
|
_NVFP4_FORMATS = ("nvfp4", "nvfp4_marlin", "nvfp4_b12x")
|
|
|
|
# Config fields come from each checkpoint's config.json (text_config for the
|
|
# multimodal wrappers); checkpoints without a local copy were recorded from
|
|
# https://huggingface.co/<name>/raw/main/config.json.
|
|
SUPPORTED_MODELS: tuple[AotModel, ...] = (
|
|
# ---- MoE checkpoints (offload expert banks) ----
|
|
AotModel(
|
|
name="Qwen/Qwen3-30B-A3B",
|
|
architecture="Qwen3MoeForCausalLM",
|
|
hidden_size=2048,
|
|
kv_groups=((4, 128),),
|
|
top_k=8,
|
|
moe_intermediate_size=768,
|
|
expert_formats=("bf16",),
|
|
aliases=("Qwen/Qwen3-30B-A3B-Thinking-2507",),
|
|
# Qwen3-VL-30B-A3B: same text tower
|
|
arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM", "Qwen3VLMoeGGUFForCausalLM"),
|
|
),
|
|
AotModel(
|
|
name="Qwen/Qwen3.5-35B-A3B",
|
|
architecture="Qwen3_5MoeForConditionalGeneration",
|
|
hidden_size=2048,
|
|
kv_groups=((2, 256),), # full-attention group; GDN layers hold no paged KV
|
|
top_k=8,
|
|
moe_intermediate_size=512,
|
|
expert_formats=("bf16",),
|
|
arch_aliases=("Qwen3_5MoeForCausalLM", "Qwen3_5MoeGGUFForCausalLM"), # text-only release of the same tower
|
|
),
|
|
AotModel(
|
|
name="Qwen/Qwen3.5-35B-A3B-FP8",
|
|
architecture="Qwen3_5MoeForConditionalGeneration",
|
|
hidden_size=2048,
|
|
kv_groups=((2, 256),),
|
|
top_k=8,
|
|
moe_intermediate_size=512,
|
|
expert_formats=("fp8_block",),
|
|
),
|
|
AotModel(
|
|
name="Qwen/Qwen3.6-35B-A3B",
|
|
architecture="Qwen3_5MoeForConditionalGeneration",
|
|
hidden_size=2048,
|
|
kv_groups=((2, 256),),
|
|
top_k=8,
|
|
moe_intermediate_size=512,
|
|
expert_formats=("bf16",),
|
|
),
|
|
AotModel(
|
|
name="Qwen/Qwen3.6-35B-A3B-FP8",
|
|
architecture="Qwen3_5MoeForConditionalGeneration",
|
|
hidden_size=2048,
|
|
kv_groups=((2, 256),),
|
|
top_k=8,
|
|
moe_intermediate_size=512,
|
|
expert_formats=("fp8_block",),
|
|
),
|
|
AotModel(
|
|
name="nvidia/Qwen3.6-35B-A3B-NVFP4",
|
|
architecture="Qwen3_5MoeForConditionalGeneration",
|
|
hidden_size=2048,
|
|
kv_groups=((2, 256),),
|
|
top_k=8,
|
|
moe_intermediate_size=512,
|
|
expert_formats=_NVFP4_FORMATS,
|
|
),
|
|
AotModel(
|
|
# QSA compressed-sparse attention (12 of 48 layers): the QSAKVCache stores K/V
|
|
# through store_cache (2 kv heads x 256 head_dim), the compressed index-key slab
|
|
# and the pending ring write via the vendored qsa triton kernels. Hyper-connections
|
|
# carry the residual, so the embedding row indexing() sees is still hidden_size.
|
|
name="RadixArk/Qwen3.8-Flash-Next-NVFP4",
|
|
architecture="Qwen4ExpForConditionalGeneration",
|
|
hidden_size=2560,
|
|
kv_groups=((2, 256),),
|
|
top_k=10,
|
|
moe_intermediate_size=640,
|
|
expert_formats=(*_NVFP4_FORMATS, "fp8_block"),
|
|
# GGUF expert rows differ per layer and reach the fused copy as runtime sizes
|
|
arch_aliases=("Qwen4ExpGGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="google/gemma-4-26B-A4B-it",
|
|
architecture="Gemma4ForConditionalGeneration",
|
|
hidden_size=2816,
|
|
# full group: num_global_key_value_heads x global_head_dim; swa group:
|
|
# num_key_value_heads x head_dim (gemma4/config.py attention_groups)
|
|
kv_groups=((2, 512), (8, 256)),
|
|
top_k=8,
|
|
moe_intermediate_size=704,
|
|
expert_formats=("bf16",),
|
|
# GGUF expert rows differ per layer and reach the fused copy as runtime sizes
|
|
arch_aliases=("Gemma4ForCausalLM", "Gemma4GGUFForCausalLM"),
|
|
),
|
|
AotModel(
|
|
name="nvidia/Gemma-4-26B-A4B-NVFP4",
|
|
architecture="Gemma4ForConditionalGeneration",
|
|
hidden_size=2816,
|
|
kv_groups=((2, 512), (8, 256)),
|
|
top_k=8,
|
|
moe_intermediate_size=704,
|
|
expert_formats=_NVFP4_FORMATS,
|
|
),
|
|
AotModel(
|
|
name="openai/gpt-oss-120b",
|
|
architecture="GptOssForCausalLM",
|
|
hidden_size=2880,
|
|
kv_groups=((8, 64),), # sliding and full layers share the same geometry
|
|
top_k=4,
|
|
moe_intermediate_size=2880,
|
|
expert_formats=("mxfp4_triton",),
|
|
arch_aliases=("GptOssGGUFForCausalLM",), # the GGUF row builds the same classes
|
|
),
|
|
AotModel(
|
|
name="openai/gpt-oss-20b",
|
|
architecture="GptOssForCausalLM",
|
|
hidden_size=2880,
|
|
kv_groups=((8, 64),),
|
|
top_k=4,
|
|
moe_intermediate_size=2880,
|
|
expert_formats=("mxfp4_triton",),
|
|
arch_aliases=("GptOssGGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="zai-org/GLM-4.7",
|
|
architecture="Glm4MoeForCausalLM",
|
|
hidden_size=5120,
|
|
kv_groups=((8, 128),),
|
|
top_k=8,
|
|
moe_intermediate_size=1536,
|
|
expert_formats=("bf16",),
|
|
arch_aliases=("Glm4MoeGGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="nvidia/GLM-4.7-NVFP4",
|
|
architecture="Glm4MoeForCausalLM",
|
|
hidden_size=5120,
|
|
kv_groups=((8, 128),),
|
|
top_k=8,
|
|
moe_intermediate_size=1536,
|
|
expert_formats=_NVFP4_FORMATS,
|
|
),
|
|
AotModel(
|
|
# MLA latent-KV: the dsa backend writes the latent via torch scatter,
|
|
# not store_cache (same as DSV4 below), so no paged-KV store groups.
|
|
name="zai-org/GLM-5.2-NVFP4",
|
|
architecture="GlmMoeDsaForCausalLM",
|
|
hidden_size=6144,
|
|
kv_groups=(),
|
|
top_k=8,
|
|
moe_intermediate_size=2048,
|
|
expert_formats=_NVFP4_FORMATS,
|
|
),
|
|
AotModel(
|
|
# GLM-5.3-Flash: hybrid KDA + NoPE-MLA/DSA (kpool indexer). Latent writes
|
|
# go through torch scatter like GLM-5.2 (no paged-KV store groups); the
|
|
# KDA conv/recurrent state lives in the LinearStatePool, not paged KV.
|
|
name="RedHatAI/GLM-5.3-Flash-NVFP4",
|
|
architecture="Glm5NextForCausalLM",
|
|
# GGUF expert rows differ per layer and reach the fused copy as runtime sizes
|
|
arch_aliases=("Glm5NextForConditionalGeneration", "Glm5NextGGUFForCausalLM"),
|
|
hidden_size=4096,
|
|
kv_groups=(),
|
|
top_k=8,
|
|
moe_intermediate_size=2048,
|
|
expert_formats=(*_NVFP4_FORMATS, "fp8_block"),
|
|
aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"),
|
|
),
|
|
AotModel(
|
|
# MiniMaxAI/MiniMax-M2.5 ships block-fp8, which has no expert-bank
|
|
# provider for this arch on main -- the NVFP4 release is the servable
|
|
# offload path, and both share the same attention/embedding shapes.
|
|
name="nvidia/MiniMax-M2.5-NVFP4",
|
|
architecture="MiniMaxM2ForCausalLM",
|
|
hidden_size=3072,
|
|
kv_groups=((8, 128),),
|
|
top_k=8,
|
|
moe_intermediate_size=1536, # experts reuse the dense intermediate_size
|
|
expert_formats=_NVFP4_FORMATS,
|
|
aliases=("MiniMaxAI/MiniMax-M2.5",),
|
|
arch_aliases=("MiniMaxM2GGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
# GQA block-sparse (BSA): the BSAKVCache stores K/V through store_cache
|
|
# (4 kv heads x 128 head_dim), the index-key slab writes via torch scatter.
|
|
# Routed experts are NVFP4 with swigluoai, which restricts the expert GEMM
|
|
# to the Triton kernels -- but the banks keep the native "nvfp4" layout,
|
|
# so its bank rows are what fast_index_copy sees.
|
|
# The upstream MiniMaxAI/MiniMax-M3 alias covers the attention/embedding
|
|
# shapes only: its experts are MXFP8, which has no expert-bank provider on
|
|
# main -- the NVFP4 release is the servable offload path (M2.5 precedent).
|
|
name="nvidia/MiniMax-M3-NVFP4",
|
|
architecture="MiniMaxM3SparseForConditionalGeneration",
|
|
hidden_size=6144,
|
|
kv_groups=((4, 128),),
|
|
top_k=4,
|
|
moe_intermediate_size=3072,
|
|
expert_formats=("nvfp4",),
|
|
aliases=("MiniMaxAI/MiniMax-M3",),
|
|
arch_aliases=("MiniMaxM3SparseForCausalLM", "MiniMaxM3GGUFForCausalLM"),
|
|
),
|
|
AotModel(
|
|
name="deepseek-ai/DeepSeek-V4-Flash",
|
|
architecture="DeepseekV4ForCausalLM",
|
|
# llama.cpp deepseek4 GGUF (unsloth 0731 GGUF; text only)
|
|
arch_aliases=("DeepseekV4GGUFForCausalLM",),
|
|
hidden_size=4096,
|
|
kv_groups=(), # MLA latent pool writes via torch scatter, not store_cache
|
|
top_k=6,
|
|
moe_intermediate_size=2048,
|
|
expert_formats=("ds_fp4",),
|
|
embed_indexing=False, # plain nn.Embedding
|
|
),
|
|
AotModel(
|
|
# DSV41 pools are packed byte rows written by the dsv41 pack kernel, not store_cache;
|
|
# the routed experts share DSV4's ds_fp4 bank layout (hidden 5120, intermediate 2304).
|
|
name="deepseek-ai/DeepSeek-V4.1-Flash",
|
|
architecture="DeepseekV41ForCausalLM",
|
|
hidden_size=5120,
|
|
kv_groups=(),
|
|
top_k=6,
|
|
moe_intermediate_size=2304,
|
|
expert_formats=("ds_fp4",),
|
|
),
|
|
AotModel(
|
|
name="RecursiveSI/flint",
|
|
architecture="FlintForCausalLM",
|
|
hidden_size=2048,
|
|
kv_groups=((4, 128),),
|
|
top_k=6,
|
|
moe_intermediate_size=512,
|
|
expert_formats=("bf16", *_NVFP4_FORMATS),
|
|
aliases=("FutureMLS-Lab/flint-nvfp4-coldgptq",),
|
|
arch_aliases=("FlintGGUFForCausalLM",),
|
|
),
|
|
# ---- dense checkpoints (store/index only, no expert banks) ----
|
|
AotModel(
|
|
name="Qwen/Qwen3.6-27B",
|
|
architecture="Qwen3_5ForConditionalGeneration",
|
|
hidden_size=5120,
|
|
kv_groups=((4, 256),),
|
|
aliases=("Qwen/Qwen3.6-27B-FP8", "nvidia/Qwen3.6-27B-NVFP4"),
|
|
arch_aliases=("Qwen3_5ForCausalLM", "Qwen3_5GGUFForCausalLM"), # text-only release of the same tower
|
|
),
|
|
AotModel(
|
|
name="google/gemma-4-12B-it",
|
|
architecture="Gemma4UnifiedForConditionalGeneration",
|
|
hidden_size=3840,
|
|
kv_groups=((1, 512), (8, 256)),
|
|
arch_aliases=("Gemma4UnifiedForCausalLM", "Gemma4UnifiedGGUFForCausalLM"),
|
|
),
|
|
AotModel(
|
|
name="nvidia/Gemma-4-31B-IT-NVFP4",
|
|
architecture="Gemma4ForConditionalGeneration",
|
|
hidden_size=5376,
|
|
kv_groups=((4, 512), (16, 256)),
|
|
),
|
|
AotModel(
|
|
name="meta-models/Muse-Glimmer-30B",
|
|
architecture="MuseGlimmerForConditionalGeneration",
|
|
hidden_size=6656,
|
|
kv_groups=((2, 128),), # sliding and full (NoPE) layers share the same geometry
|
|
aliases=("RedHatAI/Muse-Glimmer-30B-NVFP4",),
|
|
arch_aliases=("MuseGlimmerGGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="ricursiveAI/Ripple-65B-Preview",
|
|
architecture="RicursiveForCausalLM",
|
|
hidden_size=5120,
|
|
kv_groups=((8, 128),), # sliding and full (NoPE) layers share the same geometry
|
|
aliases=("ricursiveAI/Ripple-65B-Preview-NVFP4",),
|
|
arch_aliases=("RicursiveGGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="meta-llama/Llama-3.1-8B-Instruct",
|
|
architecture="LlamaForCausalLM",
|
|
hidden_size=4096,
|
|
kv_groups=((8, 128),),
|
|
arch_aliases=("LlamaGGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="Qwen/Qwen2-7B",
|
|
architecture="Qwen2ForCausalLM",
|
|
hidden_size=3584,
|
|
kv_groups=((4, 128),),
|
|
arch_aliases=("Qwen2GGUFForCausalLM",),
|
|
),
|
|
AotModel(
|
|
name="Qwen/Qwen3-8B",
|
|
architecture="Qwen3ForCausalLM",
|
|
hidden_size=4096,
|
|
kv_groups=((8, 128),),
|
|
# Qwen3-VL: same text tower
|
|
arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM", "Qwen3VLGGUFForCausalLM"),
|
|
),
|
|
AotModel(
|
|
name="mistralai/Mistral-7B-Instruct-v0.3",
|
|
architecture="MistralForCausalLM",
|
|
hidden_size=4096,
|
|
kv_groups=((8, 128),),
|
|
),
|
|
AotModel(
|
|
name="mistralai/Mistral-Small-3.1-24B-Instruct-2503",
|
|
architecture="Mistral3ForConditionalGeneration",
|
|
hidden_size=5120,
|
|
kv_groups=((8, 128),),
|
|
),
|
|
)
|
|
|
|
|
|
def store_element_sizes(model: AotModel) -> set[int]:
|
|
return {kv * hd * KV_CACHE_DTYPE_BYTES for kv, hd in model.kv_groups}
|
|
|
|
|
|
def index_variants(model: AotModel) -> set[tuple[int, int]]:
|
|
if not model.embed_indexing:
|
|
return set()
|
|
element_size = model.hidden_size * EMBED_DTYPE_BYTES
|
|
return {(element_size, num_splits_for(element_size))}
|
|
|
|
|
|
def fast_index_copy_feature_sizes(model: AotModel) -> set[int]:
|
|
sizes: set[int] = set()
|
|
for fmt in model.expert_formats:
|
|
assert model.moe_intermediate_size is not None
|
|
sizes.update(
|
|
expert_bank_row_bytes(fmt, model.hidden_size, model.moe_intermediate_size).values()
|
|
)
|
|
return sizes
|
|
|
|
|
|
|
|
def aggregate_store_element_sizes() -> tuple[int, ...]:
|
|
sizes: set[int] = set()
|
|
for model in SUPPORTED_MODELS:
|
|
sizes.update(store_element_sizes(model))
|
|
return tuple(sorted(sizes))
|
|
|
|
|
|
def aggregate_index_variants() -> tuple[tuple[int, int], ...]:
|
|
variants: set[tuple[int, int]] = set()
|
|
for model in SUPPORTED_MODELS:
|
|
variants.update(index_variants(model))
|
|
return tuple(sorted(variants))
|
|
|
|
|
|
# Synthetic per-bank sizes tests/moe/test_fused_copy.py exercises (its FEATS list);
|
|
# prebuilt so the suite also runs under FREETOKEN_DISABLE_JIT=1 release validation.
|
|
TEST_FEATURE_SIZES = (512, 8192)
|
|
|
|
|
|
def aggregate_fast_index_copy_feature_sizes() -> tuple[int, ...]:
|
|
sizes: set[int] = set(TEST_FEATURE_SIZES)
|
|
for model in SUPPORTED_MODELS:
|
|
sizes.update(fast_index_copy_feature_sizes(model))
|
|
# the per-bank kernel copies rows in fixed 128-byte steps; other sizes cannot compile
|
|
return tuple(sorted(size for size in sizes if size % 128 == 0))
|
|
|
|
|
|
__all__ = [
|
|
"AotModel",
|
|
"SUPPORTED_MODELS",
|
|
"TEST_FEATURE_SIZES",
|
|
"aggregate_fast_index_copy_feature_sizes",
|
|
"aggregate_index_variants",
|
|
"aggregate_store_element_sizes",
|
|
"expert_bank_row_bytes",
|
|
"fast_index_copy_feature_sizes",
|
|
"index_variants",
|
|
"store_element_sizes",
|
|
]
|