mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* perf(gemma4): read the source layer's kv slab on kv-shared layers * refactor(gemma4): read kv sharing from kv_shared_layers in the module and the draft view Gemma4Attention takes its source and shared flags from config.kv_shared_layers, the list the pool maps, and keys shared_kv by the source layer. The draft's read-only view routes through the same list. A group whose shared layers have no earlier layer of their type is refused at parse time. * test(gemma4): give the tiny shared-layer configs an earlier source layer * test(gemma4): drop the two tests of branches that already fail at load
333 lines
12 KiB
Python
333 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Protocol
|
|
|
|
from freetoken.utils import Registry
|
|
|
|
if TYPE_CHECKING:
|
|
import torch
|
|
from freetoken.models import ModelConfig
|
|
|
|
from .base import (
|
|
BaseCacheHandle,
|
|
BaseKVCachePool,
|
|
BasePrefixCache,
|
|
MatchResult,
|
|
SizeInfo,
|
|
)
|
|
from .format import NONE, KVFormat
|
|
|
|
|
|
class CacheManagerCreator(Protocol):
|
|
def __call__(self, device: torch.device) -> BasePrefixCache: ...
|
|
|
|
|
|
SUPPORTED_CACHE_MANAGER = Registry[CacheManagerCreator]("Cache Manager")
|
|
|
|
|
|
def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:
|
|
"""attn_type -> KV pool family, the dispatch shared by ``create_kv_pool`` and the
|
|
engine's pre-pool sizing calls (the classmethod cost/solve surface). Driven by the
|
|
group-spec walk (same source as the backend capability matrix); getattr fallbacks
|
|
cover duck-typed test configs that don't implement it."""
|
|
from freetoken.attention import AttnType
|
|
|
|
specs_fn = getattr(model_config, "kv_cache_group_specs", None)
|
|
if specs_fn is None:
|
|
if getattr(model_config, "dsv4_args", None) is not None:
|
|
from .dsv4.v4_pool import DSV4PagedKVCache
|
|
|
|
return DSV4PagedKVCache
|
|
from .mha_pool import MHAKVCache
|
|
|
|
return MHAKVCache
|
|
specs = list(specs_fn())
|
|
types = {spec.attn_type for spec in specs}
|
|
if AttnType.DSV4 in types:
|
|
from .dsv4.v4_pool import DSV4PagedKVCache
|
|
|
|
return DSV4PagedKVCache
|
|
if AttnType.DSV41 in types:
|
|
from .dsv4.v41_pool import DSV41PagedKVCache
|
|
|
|
return DSV41PagedKVCache
|
|
if AttnType.SWA in types:
|
|
from .hybrid_swa_pool import HybridSWAKVCache
|
|
|
|
return HybridSWAKVCache
|
|
if AttnType.DSA in types:
|
|
# kpool-compressed indexer (glm5_next): shadow slab + tail rings.
|
|
if any(s.attn_type == AttnType.DSA and s.index_ratio > 1 for s in specs):
|
|
from .dsa_pool import KpoolDSAKVCache
|
|
|
|
return KpoolDSAKVCache
|
|
from .dsa_pool import DSAKVCache
|
|
|
|
return DSAKVCache
|
|
if AttnType.MLA in types:
|
|
from .dsa_pool import MLAKVCache
|
|
|
|
return MLAKVCache
|
|
if AttnType.QSA in types:
|
|
from .qsa_pool import QSAKVCache
|
|
|
|
return QSAKVCache
|
|
if AttnType.BSA in types:
|
|
from .bsa_pool import BSAKVCache
|
|
|
|
return BSAKVCache
|
|
from .mha_pool import MHAKVCache
|
|
|
|
return MHAKVCache
|
|
|
|
|
|
def create_kv_pool(config, num_pages: int, device: torch.device, dtype: torch.dtype):
|
|
"""Build the engine's KV pool for ``num_pages`` USABLE pages (the dummy page and every
|
|
secondary tier -- window pool, index slab, state rings -- are derived here or inside
|
|
the pool). Single factory entry for all pool families, DSV4 included."""
|
|
from .dsv4.v4_cost_model import _dsv4_pool_sizes
|
|
from .hybrid_swa_pool import _naive_swa_num_tokens, _swa_paged_num_tokens
|
|
from .dsv4.v4_pool import DSV4PagedKVCache
|
|
|
|
model_config = config.model_config
|
|
if resolve_pool_class(model_config) is DSV4PagedKVCache:
|
|
# DSV4 is driven by the generic CacheManager over the shared page table; the pool is
|
|
# the only DSV4-specific piece (the swa_pool plug-in: window tier + cmp/idx/state
|
|
# shadows). Sizing reads dsv4_args, never the group spec.
|
|
pool = DSV4PagedKVCache(
|
|
sizes=_dsv4_pool_sizes(config, num_pages + 1), # +1 for dummy page
|
|
args=model_config.dsv4_args,
|
|
device=device,
|
|
dtype=dtype,
|
|
P=model_config.dsv4_args.window_size,
|
|
n_scratch=config.max_running_req + 1,
|
|
)
|
|
pool._init_paged_state(config.max_running_req, config.cache_type != "naive")
|
|
return pool
|
|
if getattr(model_config, "dsv41_args", None) is not None:
|
|
from .dsv4.v41_cost_model import _dsv41_pool_sizes
|
|
from .dsv4.v41_pool import DSV41PagedKVCache
|
|
|
|
# Same route as DSV4: the generic CacheManager over the shared page table; the pool is the
|
|
# swa_pool plug-in (window tier + per-source packed main / index pools + state rings).
|
|
pool = DSV41PagedKVCache(
|
|
sizes=_dsv41_pool_sizes(config, num_pages + 1), # +1 for dummy page
|
|
args=model_config.dsv41_args,
|
|
device=device,
|
|
dtype=dtype,
|
|
n_scratch=config.max_running_req + 1,
|
|
)
|
|
pool._init_paged_state(config.max_running_req, config.cache_type != "naive")
|
|
return pool
|
|
|
|
num_swa_tokens = None
|
|
# Both the naive and radix SWA paths share the global-paged swa pool; radix sizes it by
|
|
# ratio (cross-request reuse), naive by concurrency x window.
|
|
if model_config.has_swa_attention:
|
|
num_swa_tokens = (
|
|
_swa_paged_num_tokens(config, num_pages + 1)
|
|
if config.cache_type == "swa_radix"
|
|
else _naive_swa_num_tokens(config)
|
|
)
|
|
return create_kvcache_pool(
|
|
model_config=model_config,
|
|
num_pages=num_pages + 1, # +1 for dummy page
|
|
page_size=config.page_size,
|
|
num_swa_tokens=num_swa_tokens,
|
|
device=device,
|
|
dtype=dtype,
|
|
num_req_slots=config.max_running_req + 1, # + 1 for the dummy request row
|
|
num_speculative_tokens=getattr(config, "decode_rows_per_req", 1) - 1,
|
|
kv_format=getattr(config, "kv_format", NONE),
|
|
)
|
|
|
|
|
|
def create_kvcache_pool(
|
|
model_config: ModelConfig,
|
|
num_pages: int,
|
|
page_size: int,
|
|
dtype: torch.dtype,
|
|
device: torch.device,
|
|
num_swa_tokens: int | None = None,
|
|
num_req_slots: int | None = None,
|
|
num_speculative_tokens: int = 0,
|
|
kv_format: KVFormat = NONE,
|
|
) -> BaseKVCachePool:
|
|
if model_config.has_swa_attention:
|
|
from .hybrid_swa_pool import HybridSWAKVCache
|
|
|
|
return HybridSWAKVCache(
|
|
groups=model_config.kv_cache_group_specs(),
|
|
num_layers=model_config.num_layers,
|
|
num_full_pages=num_pages,
|
|
page_size=page_size,
|
|
num_swa_tokens=num_swa_tokens,
|
|
device=device,
|
|
dtype=dtype,
|
|
kv_format=kv_format,
|
|
)
|
|
|
|
from .mha_pool import MHAKVCache
|
|
|
|
# Hybrid linear-attention models only store paged KV for their non-linear
|
|
# layers; the linear layers keep a separate recurrent state (LinearStatePool).
|
|
# The linear group emits no paged spec, so the remaining paged spec(s) drive
|
|
# the dispatch below; the paged pool backs JUST those layers via a global-id
|
|
# -> dense-slot remap (layer_ids) so the (majority) linear layers cost no
|
|
# slabs.
|
|
layer_ids: tuple[int, ...] | None = None
|
|
kv_specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0]
|
|
if any(s.shared_layers for s in kv_specs):
|
|
raise NotImplementedError("KV-shared layers are only mapped by the hybrid full/SWA pool")
|
|
if model_config.has_linear_attention:
|
|
assert len(kv_specs) == 1, (
|
|
f"hybrid-linear models support one paged-KV group, got "
|
|
f"{[s.name for s in kv_specs]}"
|
|
)
|
|
layer_ids = kv_specs[0].layer_ids
|
|
|
|
# Latent-KV MLA / GQA block-sparse models declare their geometry on the single
|
|
# paged spec; the same spec fields drive the KV cost model, so the factory and
|
|
# the budget can never disagree.
|
|
from freetoken.attention import AttnType as _AttnType
|
|
|
|
if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA:
|
|
from .bsa_pool import BSAKVCache
|
|
|
|
spec = kv_specs[0]
|
|
assert layer_ids is None, "hybrid-linear x BSA has no pool support yet"
|
|
return BSAKVCache(
|
|
num_kv_heads=spec.num_kv_heads,
|
|
num_layers=model_config.num_layers,
|
|
head_dim=spec.head_dim,
|
|
num_pages=num_pages,
|
|
page_size=page_size,
|
|
dtype=dtype,
|
|
device=device,
|
|
index_head_dim=spec.index_head_dim,
|
|
num_index_layers=spec.num_index_layers,
|
|
kv_format=kv_format,
|
|
)
|
|
|
|
# QSA (Qwen3.8-Flash-Next): the same GQA group, but it stores one index key per
|
|
# index_ratio tokens and adds per-request tiers sized by the concurrency, not the pages.
|
|
# layer_ids is mandatory here -- the model is hybrid-linear, and letting MHAKVCache back
|
|
# all num_layers would allocate K/V slabs for the GDN layers too.
|
|
if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.QSA:
|
|
from .qsa_pool import QSAKVCache
|
|
|
|
spec = kv_specs[0]
|
|
if num_req_slots is None:
|
|
raise ValueError("QSA pools need num_req_slots (max_running_req + 1)")
|
|
return QSAKVCache(
|
|
num_kv_heads=spec.num_kv_heads,
|
|
num_layers=model_config.num_layers,
|
|
head_dim=spec.head_dim,
|
|
num_pages=num_pages,
|
|
page_size=page_size,
|
|
dtype=dtype,
|
|
device=device,
|
|
index_head_dim=spec.index_head_dim,
|
|
num_index_layers=spec.num_index_layers,
|
|
index_ratio=spec.index_ratio,
|
|
num_req_slots=num_req_slots,
|
|
ring_capacity=QSAKVCache.ring_capacity_for(spec.index_ratio, num_speculative_tokens),
|
|
layer_ids=spec.layer_ids,
|
|
mrope=model_config.model_is_mrope,
|
|
kv_format=kv_format,
|
|
)
|
|
|
|
if len(kv_specs) == 1 and kv_specs[0].mla:
|
|
from .dsa_pool import DSAKVCache, KpoolDSAKVCache, MLAKVCache
|
|
|
|
spec = kv_specs[0]
|
|
# With a layer remap the pool allocates len(layer_ids) slabs; without one
|
|
# it backs every model layer (all-MLA models, GLM-5.2).
|
|
num_layers = model_config.num_layers if layer_ids is None else len(layer_ids)
|
|
if spec.index_head_dim > 0 and spec.num_index_layers > 0:
|
|
common = dict(
|
|
latent_dim=spec.head_dim,
|
|
num_layers=num_layers,
|
|
num_pages=num_pages,
|
|
page_size=page_size,
|
|
dtype=dtype,
|
|
device=device,
|
|
index_head_dim=spec.index_head_dim,
|
|
num_index_layers=spec.num_index_layers,
|
|
layer_ids=layer_ids,
|
|
kv_format=kv_format,
|
|
)
|
|
if spec.index_ratio > 1:
|
|
# kpool tail rings are keyed by Req.table_idx; + 1 covers the dummy request row.
|
|
if num_req_slots is None:
|
|
raise ValueError("kpool pools need num_req_slots (max_running_req + 1)")
|
|
return KpoolDSAKVCache(
|
|
**common,
|
|
index_ratio=spec.index_ratio,
|
|
num_req_slots=num_req_slots,
|
|
)
|
|
return DSAKVCache(**common)
|
|
return MLAKVCache(
|
|
latent_dim=spec.head_dim,
|
|
num_layers=num_layers,
|
|
num_pages=num_pages,
|
|
page_size=page_size,
|
|
dtype=dtype,
|
|
device=device,
|
|
layer_ids=layer_ids,
|
|
kv_format=kv_format,
|
|
)
|
|
|
|
spec = kv_specs[0] if len(kv_specs) == 1 else None
|
|
return MHAKVCache(
|
|
num_kv_heads=spec.num_kv_heads if spec is not None else model_config.num_kv_heads,
|
|
num_pages=num_pages,
|
|
page_size=page_size,
|
|
num_layers=model_config.num_layers,
|
|
head_dim=spec.head_dim if spec is not None else model_config.head_dim,
|
|
device=device,
|
|
dtype=dtype,
|
|
layer_ids=layer_ids,
|
|
kv_format=kv_format,
|
|
)
|
|
|
|
|
|
@SUPPORTED_CACHE_MANAGER.register("naive")
|
|
def create_naive_cache(device: torch.device, page_size: int | None = None):
|
|
from .naive_cache import NaivePrefixCache
|
|
|
|
return NaivePrefixCache(device=device) # naive has no page arithmetic
|
|
|
|
|
|
@SUPPORTED_CACHE_MANAGER.register("radix")
|
|
def create_radix_cache(device: torch.device, page_size: int | None = None):
|
|
from .radix_cache import RadixPrefixCache
|
|
|
|
return RadixPrefixCache(device=device, page_size=page_size)
|
|
|
|
|
|
# NOTE: "hybrid_radix" is NOT registered as a user-facing --cache-type. It is the internal
|
|
# materialization of "radix" for hybrid GDN models (cross-request GDN-state reuse), produced by
|
|
# _resolve_cache_type and built directly in CacheManager._make_prefix_cache (HybridRadixCache
|
|
# needs page_size). Users pick "radix" (the concept) or "naive"; the engine picks hybrid_radix.
|
|
|
|
|
|
def create_prefix_cache(
|
|
device: torch.device, type: str, page_size: int | None = None
|
|
) -> BasePrefixCache:
|
|
return SUPPORTED_CACHE_MANAGER[type](device, page_size=page_size)
|
|
|
|
|
|
__all__ = [
|
|
"create_kv_pool",
|
|
"create_kvcache_pool",
|
|
"create_prefix_cache",
|
|
"resolve_pool_class",
|
|
"BaseKVCachePool",
|
|
"BaseCacheHandle",
|
|
"BasePrefixCache",
|
|
"SizeInfo",
|
|
"MatchResult",
|
|
"SUPPORTED_CACHE_MANAGER",
|
|
]
|