Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

177 lines
7.4 KiB
Python

"""Runtime metrics for /v1/stats. The FrontendManager owns one StatsTracker and feeds it
every UserReply (the single chokepoint in listen()). kv/mamba/vram keep their last-known-value
semantics like ShellStats; throughput uses an independent sliding-window rate (NOT cumulative
average like tok_s), so idle polls decay to zero by wall clock."""
from __future__ import annotations
import time
from collections import deque
from typing import Any
class StatsTracker:
def __init__(self, window_s: float = 5.0) -> None:
self.window_s = window_s
# maxlen bounds memory on the headless path: stale-sample eviction is poll-driven
# (only _rate() trims to window_s), and clients that never hit /v1/stats (e.g.
# codex/claude via /v1/chat/completions) would otherwise grow these unbounded.
# 4096 is generous vs the sliding window's span at any realistic reply rate.
self._decode: "deque[tuple[float, int]]" = deque(maxlen=4096)
self._prefill: "deque[tuple[float, int]]" = deque(maxlen=4096)
self._inflight: set[int] = set()
# Requests for which an abort was dispatched but the scheduler's explicit terminal
# acknowledgement has not arrived yet. They remain active until that barrier, while
# any sampled tokens racing the abort continue to count toward lifetime totals.
self._aborting: set[int] = set()
self.completed = 0
# Cumulative prompt/completion tokens since this process started (lifetime for THIS served
# model). Exposed in /v1/stats so the desktop can diff consecutive polls into per-model
# "cost saved by running locally" accounting. Monotonic; resets when the process restarts.
self.prompt_tokens_total = 0
self.completion_tokens_total = 0
self.kv_used_pages = 0
self.kv_total_pages = 0
self.mamba_used_slots = 0
self.mamba_total_slots = 0
self.swa_used_tokens = 0
self.swa_total_tokens = 0
self.vram_bytes = 0
@property
def active(self) -> int:
return len(self._inflight)
@property
def inflight_uids(self) -> tuple[int, ...]:
"""Stable snapshot used by prepare-stop to abort every still-admitted request."""
return tuple(sorted(self._inflight))
def on_new_user(self, uid: int) -> None:
self._inflight.add(uid)
self._aborting.discard(uid)
def on_abort(self, uid: int) -> None:
if uid in self._inflight:
self._aborting.add(uid)
def observe(self, reply: Any, now: float | None = None) -> None:
t = time.monotonic() if now is None else now
if getattr(reply, "completion_tokens_delta", 0) > 0:
self._decode.append((t, reply.completion_tokens_delta))
self.completion_tokens_total += reply.completion_tokens_delta
if getattr(reply, "prompt_tokens_delta", 0) > 0:
self._prefill.append((t, reply.prompt_tokens_delta))
self.prompt_tokens_total += reply.prompt_tokens_delta
if getattr(reply, "kv_total_pages", 0) > 0: # ignore 0/0 (prompt reply, owned-KV)
self.kv_used_pages = reply.kv_used_pages
self.kv_total_pages = reply.kv_total_pages
if getattr(reply, "mamba_total_slots", 0) > 0: # hybrid (GDN) only
self.mamba_used_slots = reply.mamba_used_slots
self.mamba_total_slots = reply.mamba_total_slots
if getattr(reply, "swa_total_tokens", 0) > 0: # SWA (window pool) only
self.swa_used_tokens = reply.swa_used_tokens
self.swa_total_tokens = reply.swa_total_tokens
if getattr(reply, "gpu_mem_bytes", 0) > 0:
self.vram_bytes = reply.gpu_mem_bytes
if getattr(reply, "finished", False):
uid = getattr(reply, "uid", None)
if uid in self._inflight:
self._inflight.discard(uid)
if uid in self._aborting:
self._aborting.discard(uid)
else:
self.completed += 1
def _rate(self, window: "deque[tuple[float, int]]", now: float | None) -> float:
t = time.monotonic() if now is None else now
cutoff = t - self.window_s
while window and window[0][0] < cutoff:
window.popleft()
if not window:
return 0.0
total = sum(n for _ts, n in window)
span = max(t - window[0][0], 1e-9)
return total / span
def decode_tps(self, now: float | None = None) -> float:
return self._rate(self._decode, now)
def prefill_tps(self, now: float | None = None) -> float:
return self._rate(self._prefill, now)
def derive_model_card(config: Any) -> dict:
"""attn enum + moe bool + ctx from the model config; input_modalities is what the API accepts right now."""
mc = config.model_config
if getattr(mc, "has_linear_attention", False):
attn = "hybrid_linear"
elif getattr(mc, "has_swa_attention", False):
attn = "hybrid_swa"
else:
attn = "mha"
return {
"id": config.served_model_name,
"ctx": config.max_seq_len,
"attn": attn,
"moe": bool(getattr(mc, "is_moe", False)),
"input_modalities": ["text", *sorted(config.served_modalities)],
}
def _swa_page_size(config: Any) -> int:
from freetoken.kvcache.cache_status import window_pool_spec
spec = window_pool_spec(config)
return spec.page_size if spec is not None else 1
def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:
"""Full /v1/stats doc. throughput is 0 when idle; kv/mamba/swa are null
when their total is 0 (owned-KV / non-hybrid / non-SWA). kv and swa share one shape:
pages + the pool's own page_size (tokens = pages x page_size). gpus: the engine's GPU as
[{index, name, uuid, total_bytes}] (the primary rank's; a list so TP can extend it), []
until the readiness meta arrives."""
tr: StatsTracker = state.stats
config = state.config
ready_at = getattr(state, "ready_at", None)
uptime_s = max(0, int(time.monotonic() - ready_at)) if ready_at is not None else 0
pools = getattr(state, "cache_pools", None) or {}
kv = (
{"used_pages": tr.kv_used_pages, "total_pages": tr.kv_total_pages,
"page_size": pools.get("page_size") or getattr(config, "page_size", 1)}
if tr.kv_total_pages > 0 else None
)
mamba = (
{"used_slots": tr.mamba_used_slots, "total_slots": tr.mamba_total_slots}
if tr.mamba_total_slots > 0 else None
)
sps = _swa_page_size(config)
swa = (
{"used_pages": tr.swa_used_tokens // sps, "total_pages": tr.swa_total_tokens // sps,
"page_size": sps}
if tr.swa_total_tokens > 0 else None
)
return {
"instance_id": getattr(state, "instance_id", None),
"model": derive_model_card(config),
"uptime_s": uptime_s,
"kv": kv,
"mamba": mamba,
"swa": swa,
"vram_bytes": tr.vram_bytes,
"gpus": list(getattr(state, "gpus", None) or []),
"throughput": {
"decode_tps": round(tr.decode_tps(), 1),
"prefill_tps": round(tr.prefill_tps(), 1),
},
"requests": {
"active": tr.active,
"completed": tr.completed,
"p95_ms": p95_ms,
"ttft_mean_ms": ttft_mean_ms,
"prompt_tokens_total": tr.prompt_tokens_total,
"completion_tokens_total": tr.completion_tokens_total,
},
}