mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
177 lines
7.4 KiB
Python
177 lines
7.4 KiB
Python
"""Runtime metrics for /v1/stats. The FrontendManager owns one StatsTracker and feeds it
|
|
every UserReply (the single chokepoint in listen()). kv/mamba/vram keep their last-known-value
|
|
semantics like ShellStats; throughput uses an independent sliding-window rate (NOT cumulative
|
|
average like tok_s), so idle polls decay to zero by wall clock."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from collections import deque
|
|
from typing import Any
|
|
|
|
|
|
class StatsTracker:
|
|
def __init__(self, window_s: float = 5.0) -> None:
|
|
self.window_s = window_s
|
|
# maxlen bounds memory on the headless path: stale-sample eviction is poll-driven
|
|
# (only _rate() trims to window_s), and clients that never hit /v1/stats (e.g.
|
|
# codex/claude via /v1/chat/completions) would otherwise grow these unbounded.
|
|
# 4096 is generous vs the sliding window's span at any realistic reply rate.
|
|
self._decode: "deque[tuple[float, int]]" = deque(maxlen=4096)
|
|
self._prefill: "deque[tuple[float, int]]" = deque(maxlen=4096)
|
|
self._inflight: set[int] = set()
|
|
# Requests for which an abort was dispatched but the scheduler's explicit terminal
|
|
# acknowledgement has not arrived yet. They remain active until that barrier, while
|
|
# any sampled tokens racing the abort continue to count toward lifetime totals.
|
|
self._aborting: set[int] = set()
|
|
self.completed = 0
|
|
# Cumulative prompt/completion tokens since this process started (lifetime for THIS served
|
|
# model). Exposed in /v1/stats so the desktop can diff consecutive polls into per-model
|
|
# "cost saved by running locally" accounting. Monotonic; resets when the process restarts.
|
|
self.prompt_tokens_total = 0
|
|
self.completion_tokens_total = 0
|
|
self.kv_used_pages = 0
|
|
self.kv_total_pages = 0
|
|
self.mamba_used_slots = 0
|
|
self.mamba_total_slots = 0
|
|
self.swa_used_tokens = 0
|
|
self.swa_total_tokens = 0
|
|
self.vram_bytes = 0
|
|
|
|
@property
|
|
def active(self) -> int:
|
|
return len(self._inflight)
|
|
|
|
@property
|
|
def inflight_uids(self) -> tuple[int, ...]:
|
|
"""Stable snapshot used by prepare-stop to abort every still-admitted request."""
|
|
return tuple(sorted(self._inflight))
|
|
|
|
def on_new_user(self, uid: int) -> None:
|
|
self._inflight.add(uid)
|
|
self._aborting.discard(uid)
|
|
|
|
def on_abort(self, uid: int) -> None:
|
|
if uid in self._inflight:
|
|
self._aborting.add(uid)
|
|
|
|
def observe(self, reply: Any, now: float | None = None) -> None:
|
|
t = time.monotonic() if now is None else now
|
|
if getattr(reply, "completion_tokens_delta", 0) > 0:
|
|
self._decode.append((t, reply.completion_tokens_delta))
|
|
self.completion_tokens_total += reply.completion_tokens_delta
|
|
if getattr(reply, "prompt_tokens_delta", 0) > 0:
|
|
self._prefill.append((t, reply.prompt_tokens_delta))
|
|
self.prompt_tokens_total += reply.prompt_tokens_delta
|
|
if getattr(reply, "kv_total_pages", 0) > 0: # ignore 0/0 (prompt reply, owned-KV)
|
|
self.kv_used_pages = reply.kv_used_pages
|
|
self.kv_total_pages = reply.kv_total_pages
|
|
if getattr(reply, "mamba_total_slots", 0) > 0: # hybrid (GDN) only
|
|
self.mamba_used_slots = reply.mamba_used_slots
|
|
self.mamba_total_slots = reply.mamba_total_slots
|
|
if getattr(reply, "swa_total_tokens", 0) > 0: # SWA (window pool) only
|
|
self.swa_used_tokens = reply.swa_used_tokens
|
|
self.swa_total_tokens = reply.swa_total_tokens
|
|
if getattr(reply, "gpu_mem_bytes", 0) > 0:
|
|
self.vram_bytes = reply.gpu_mem_bytes
|
|
if getattr(reply, "finished", False):
|
|
uid = getattr(reply, "uid", None)
|
|
if uid in self._inflight:
|
|
self._inflight.discard(uid)
|
|
if uid in self._aborting:
|
|
self._aborting.discard(uid)
|
|
else:
|
|
self.completed += 1
|
|
|
|
def _rate(self, window: "deque[tuple[float, int]]", now: float | None) -> float:
|
|
t = time.monotonic() if now is None else now
|
|
cutoff = t - self.window_s
|
|
while window and window[0][0] < cutoff:
|
|
window.popleft()
|
|
if not window:
|
|
return 0.0
|
|
total = sum(n for _ts, n in window)
|
|
span = max(t - window[0][0], 1e-9)
|
|
return total / span
|
|
|
|
def decode_tps(self, now: float | None = None) -> float:
|
|
return self._rate(self._decode, now)
|
|
|
|
def prefill_tps(self, now: float | None = None) -> float:
|
|
return self._rate(self._prefill, now)
|
|
|
|
|
|
def derive_model_card(config: Any) -> dict:
|
|
"""attn enum + moe bool + ctx from the model config; input_modalities is what the API accepts right now."""
|
|
mc = config.model_config
|
|
if getattr(mc, "has_linear_attention", False):
|
|
attn = "hybrid_linear"
|
|
elif getattr(mc, "has_swa_attention", False):
|
|
attn = "hybrid_swa"
|
|
else:
|
|
attn = "mha"
|
|
return {
|
|
"id": config.served_model_name,
|
|
"ctx": config.max_seq_len,
|
|
"attn": attn,
|
|
"moe": bool(getattr(mc, "is_moe", False)),
|
|
"input_modalities": ["text", *sorted(config.served_modalities)],
|
|
}
|
|
|
|
|
|
def _swa_page_size(config: Any) -> int:
|
|
from freetoken.kvcache.cache_status import window_pool_spec
|
|
|
|
spec = window_pool_spec(config)
|
|
return spec.page_size if spec is not None else 1
|
|
|
|
|
|
def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:
|
|
"""Full /v1/stats doc. throughput is 0 when idle; kv/mamba/swa are null
|
|
when their total is 0 (owned-KV / non-hybrid / non-SWA). kv and swa share one shape:
|
|
pages + the pool's own page_size (tokens = pages x page_size). gpus: the engine's GPU as
|
|
[{index, name, uuid, total_bytes}] (the primary rank's; a list so TP can extend it), []
|
|
until the readiness meta arrives."""
|
|
tr: StatsTracker = state.stats
|
|
config = state.config
|
|
ready_at = getattr(state, "ready_at", None)
|
|
uptime_s = max(0, int(time.monotonic() - ready_at)) if ready_at is not None else 0
|
|
pools = getattr(state, "cache_pools", None) or {}
|
|
kv = (
|
|
{"used_pages": tr.kv_used_pages, "total_pages": tr.kv_total_pages,
|
|
"page_size": pools.get("page_size") or getattr(config, "page_size", 1)}
|
|
if tr.kv_total_pages > 0 else None
|
|
)
|
|
mamba = (
|
|
{"used_slots": tr.mamba_used_slots, "total_slots": tr.mamba_total_slots}
|
|
if tr.mamba_total_slots > 0 else None
|
|
)
|
|
sps = _swa_page_size(config)
|
|
swa = (
|
|
{"used_pages": tr.swa_used_tokens // sps, "total_pages": tr.swa_total_tokens // sps,
|
|
"page_size": sps}
|
|
if tr.swa_total_tokens > 0 else None
|
|
)
|
|
return {
|
|
"instance_id": getattr(state, "instance_id", None),
|
|
"model": derive_model_card(config),
|
|
"uptime_s": uptime_s,
|
|
"kv": kv,
|
|
"mamba": mamba,
|
|
"swa": swa,
|
|
"vram_bytes": tr.vram_bytes,
|
|
"gpus": list(getattr(state, "gpus", None) or []),
|
|
"throughput": {
|
|
"decode_tps": round(tr.decode_tps(), 1),
|
|
"prefill_tps": round(tr.prefill_tps(), 1),
|
|
},
|
|
"requests": {
|
|
"active": tr.active,
|
|
"completed": tr.completed,
|
|
"p95_ms": p95_ms,
|
|
"ttft_mean_ms": ttft_mean_ms,
|
|
"prompt_tokens_total": tr.prompt_tokens_total,
|
|
"completion_tokens_total": tr.completion_tokens_total,
|
|
},
|
|
}
|