mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
312 lines
14 KiB
Python
312 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from typing import TYPE_CHECKING, List, Literal, Tuple
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.attention import BaseAttnBackend, BaseAttnMetadata
|
|
from freetoken.attention.linear import FLAMetadata
|
|
from freetoken.kvcache import BaseCacheHandle, BaseKVCachePool
|
|
from freetoken.kvcache.linear_state_pool import LinearStatePool
|
|
from freetoken.models.blocks import FeatureSink
|
|
|
|
|
|
@dataclass
|
|
class SamplingParams:
|
|
temperature: float = 0.0
|
|
top_k: int = -1
|
|
top_p: float = 1.0
|
|
ignore_eos: bool = False
|
|
max_tokens: int = 1024
|
|
# Stop strings (OpenAI `stop` / Anthropic `stop_sequences`). Generation finishes when one
|
|
# appears in the decoded output; the matched substring (and anything after) is trimmed.
|
|
stop_strs: list[str] = field(default_factory=list)
|
|
presence_penalty: float = 0.0
|
|
frequency_penalty: float = 0.0
|
|
# Stamp the ReqStats snapshot on every reply, not just the terminal one (llama.cpp's
|
|
# timings_per_token; the API layer has already ANDed it with --enable-timings-report).
|
|
timings_per_token: bool = False
|
|
|
|
@property
|
|
def is_greedy(self) -> bool:
|
|
return self.temperature <= 0.0 or self.top_k == 1
|
|
|
|
|
|
@dataclass
|
|
class ReqStats:
|
|
"""Per-request timing and draft counters, stamped by the scheduler with
|
|
time.perf_counter(). Spans follow llama.cpp's server_slot_stats plus a queue
|
|
span: queue = waiting queue -> first prefill chunk launch, prompt = first
|
|
chunk launch -> first sampled token on the host (chunk gaps included),
|
|
predicted = first token -> last committed token. One instance is shared by
|
|
every chunk Req of a request, so the stamps survive chunked prefill."""
|
|
|
|
enqueued_at: float | None = None # entered the scheduler's waiting queue
|
|
first_chunk_at: float | None = None # first prefill chunk launched
|
|
first_token_at: float | None = None # first sampled token back on the host
|
|
last_token_at: float | None = None # last committed token
|
|
cache_n: int = 0 # prompt tokens served from the prefix cache at admission
|
|
prompt_n: int = 0 # prompt tokens actually computed
|
|
draft_n: int = 0 # drafted tokens summed over verify steps
|
|
draft_n_accepted: int = 0 # accepted drafts that were committed, after stop truncation
|
|
|
|
def stamp_token(self, now: float) -> None:
|
|
if self.first_token_at is None:
|
|
self.first_token_at = now
|
|
self.last_token_at = now
|
|
|
|
def finish(self, completion_tokens: int) -> dict | None:
|
|
"""Raw spans and counts for the terminal reply; None when no prefill ever
|
|
launched or no token was committed (errors and aborts emit nothing)."""
|
|
if self.first_chunk_at is None or self.first_token_at is None:
|
|
return None
|
|
return {
|
|
"queue_ms": (self.first_chunk_at - self.enqueued_at) * 1000.0
|
|
if self.enqueued_at is not None else 0.0,
|
|
"cache_n": self.cache_n,
|
|
"prompt_n": self.prompt_n,
|
|
"prompt_ms": (self.first_token_at - self.first_chunk_at) * 1000.0,
|
|
"predicted_n": completion_tokens,
|
|
# 1 us floor as in llama.cpp: the first token can land in the same instant
|
|
"predicted_ms": max((self.last_token_at - self.first_token_at) * 1000.0, 0.001),
|
|
"draft_n": self.draft_n,
|
|
"draft_n_accepted": self.draft_n_accepted,
|
|
}
|
|
|
|
|
|
@dataclass(eq=False)
|
|
class Req:
|
|
input_ids: torch.Tensor # cpu tensor
|
|
table_idx: int
|
|
cached_len: int
|
|
output_len: int
|
|
uid: int
|
|
sampling_params: SamplingParams
|
|
cache_handle: BaseCacheHandle
|
|
# per-item processor outputs and the tokenizer's precomputed mrope rows and delta
|
|
mm_items: list | None = None
|
|
mrope_positions_full: torch.Tensor | None = None # [3, prompt_len] int32, CPU
|
|
mrope_delta: int = 0
|
|
# The whole prompt's length; 0 = this request's input_ids (a ChunkedReq's end at its chunk).
|
|
prompt_len: int = 0
|
|
|
|
# --- hybrid-radix (GDN linear-state) per-request slots; None for non-hybrid models or
|
|
# until allocated from LinearStatePool. Set by the scheduler (P2). ---
|
|
linear_slot_idx: int | None = None # live GDN state slot (sglang mamba_pool_idx)
|
|
mamba_ping_pong: tuple[int, int] | None = None # 2 donatable track slots under overlap
|
|
mamba_next_track_idx: int = 0 # which ping-pong slot is the next snapshot dst (0/1)
|
|
mamba_last_track_seqlen: int | None = None # chunk-aligned committed len of the last snapshot
|
|
mamba_restore_src: int | None = None # on a prefix hit: tree snapshot slot to COW into the live slot (first chunk only)
|
|
swa_evicted_seqlen: int = 0 # SWA radix: positions < this had their swa KV freed (slid out of window) during decode
|
|
decode_batch_idx: int = 0 # SWA radix: # of decode forwards done; the proactive free_swa skips the first (overlap guard)
|
|
# Set once, at the first sampled tool-call opener token (scheduler detection): the state
|
|
# length just after that token (its index + 1). A client-side rewrite of the echoed tool
|
|
# call diverges strictly after this point, so it is the deepest reuse boundary that
|
|
# survives such a rewrite. GDN: the state is frozen into a ping-pong slot when cached_len
|
|
# reaches it (snapshot_toolcall_anchor) and donated at finish. SWA: caps the proactive
|
|
# out-of-window eviction so the window ending here stays resumable.
|
|
toolcall_anchor_len: int | None = None
|
|
# Abort arrived while this request's forward was in flight (overlap scheduling). The abort
|
|
# handler must not free resources under an in-flight forward; it sets this flag and
|
|
# _process_last_data frees the request when the batch drains (after copy_done.synchronize).
|
|
aborted: bool = False
|
|
# Timing/draft counters shared with every other chunk Req of this request (set by
|
|
# the PrefillManager from the PendingReq's instance; a default blank otherwise).
|
|
stats: ReqStats = field(default_factory=ReqStats, init=False, repr=False)
|
|
output_token_counts: torch.Tensor | None = field(default=None, init=False, repr=False)
|
|
|
|
def __post_init__(self) -> None:
|
|
assert self.input_ids.is_cpu
|
|
self.device_len = len(self.input_ids)
|
|
self.max_device_len = len(self.input_ids) + self.output_len
|
|
self.prompt_len = self.prompt_len or self.device_len
|
|
assert 0 <= self.cached_len < self.device_len <= self.max_device_len
|
|
self._alloc_ids_buf()
|
|
|
|
def _alloc_ids_buf(self) -> None:
|
|
self._ids_buf = torch.empty(self.max_device_len, dtype=self.input_ids.dtype)
|
|
self._ids_buf[: self.device_len] = self.input_ids
|
|
self.input_ids = self._ids_buf[: self.device_len]
|
|
|
|
@property
|
|
def remain_len(self) -> int:
|
|
return self.max_device_len - self.device_len
|
|
|
|
@property
|
|
def extend_len(self) -> int:
|
|
return self.device_len - self.cached_len
|
|
|
|
def complete_one(self) -> None:
|
|
self.cached_len = self.device_len
|
|
self.device_len += 1
|
|
|
|
def widen_pending(self, k: int) -> None:
|
|
"""Extend the pending token at cached_len by k provisional draft tokens."""
|
|
self.device_len += k
|
|
|
|
def commit_pending(self, n: int) -> None:
|
|
"""Make n tokens final and leave the last committed one pending at cached_len."""
|
|
self.cached_len += n
|
|
self.device_len = self.cached_len + 1
|
|
|
|
def append_host(self, next_token: torch.Tensor) -> None:
|
|
n = self.input_ids.numel()
|
|
m = n + next_token.numel()
|
|
assert m <= self.max_device_len
|
|
self._ids_buf[n:m] = next_token
|
|
self.input_ids = self._ids_buf[:m]
|
|
|
|
@property
|
|
def can_decode(self) -> bool:
|
|
return self.remain_len > 0
|
|
|
|
def __repr__(self) -> str:
|
|
return (
|
|
f"{type(self)}(table_idx={self.table_idx}, "
|
|
f"cached_len={self.cached_len}, device_len={self.device_len}, "
|
|
f"max_device_len={self.max_device_len})"
|
|
)
|
|
|
|
|
|
|
|
@dataclass
|
|
class Batch:
|
|
reqs: List[Req]
|
|
phase: Literal["prefill", "decode", "verify"]
|
|
# query rows per request: 1 on decode, more on verify, None on prefill (ragged)
|
|
rows_per_req: int | None = None
|
|
# these fields should be set by scheduler
|
|
input_ids: torch.Tensor = field(init=False)
|
|
positions: torch.Tensor = field(init=False)
|
|
# [3, n] t/h/w rope positions on mrope models; positions keeps its sequence-index meaning for token_pool / page_table
|
|
mrope_positions: torch.Tensor | None = field(default=None, init=False)
|
|
out_loc: torch.Tensor | None = field(init=False)
|
|
# Per-forward GatedDeltaNet metadata (cu_seqlens / cache_indices / continuation
|
|
# flags), built once and shared by all GDN layers. Lazily built by the GDN op if
|
|
# the scheduler/graph didn't set it.
|
|
fla_metadata: "FLAMetadata | None" = field(default=None, init=False)
|
|
padded_reqs: List[Req] = field(init=False)
|
|
# DSV4 paged-KV out-locations for this batch (None for non-DSV4 models). Set by the scheduler.
|
|
# This decode batch's padded per-row page-table rows. Attention backends that must read
|
|
# positions anywhere in a request's history snapshot those rows before a captured replay
|
|
# (DSV4), since the next batch's allocate_paged mutates the live table.
|
|
active_table_idx: "torch.Tensor | None" = None
|
|
# this field should be set by attention backend
|
|
attn_metadata: BaseAttnMetadata = field(init=False)
|
|
# concatenated multimodal soft-token embeddings for a prefill batch (or None) and the batch rows they land on
|
|
mm_embeds: torch.Tensor | None = field(default=None, init=False)
|
|
mm_rows: torch.Tensor | None = field(default=None, init=False)
|
|
# per batch token, the end (exclusive, in its request) of the image span holding it, 0 for text: the block a bidirectional layer attends within
|
|
mm_block_ends: torch.Tensor | None = field(default=None, init=False)
|
|
# this chunk's cache-miss items to encode and the gather plan [(uid, hash, row_lo, row_hi, n, pos), ...] in scatter order
|
|
mm_encoder_jobs: list | None = field(default=None, init=False)
|
|
mm_gather_plan: list | None = field(default=None, init=False)
|
|
# Prefill log stats snapshotted at schedule time (before forward's complete_one()
|
|
# advances cached_len), so the prefill log reports the tokens actually forwarded and
|
|
# the prefix-cache hit -- matching SGLang's #new-token / #cached-token. Set by the
|
|
# PrefillManager; 0 on decode batches.
|
|
log_new_tokens: int = field(default=0, init=False)
|
|
log_cached_tokens: int = field(default=0, init=False)
|
|
# (uid, complete prompt length, prefix-cache hit) for requests entering their first
|
|
# prepared prefill batch. The scheduler turns these into PromptAdmittedMsg only AFTER
|
|
# _prepare_batch succeeds. Continuation chunks leave this empty, so accounting is
|
|
# exactly-once.
|
|
prompt_admissions: List[Tuple[int, int, int]] = field(default_factory=list, init=False)
|
|
# per-row hidden states a draft model reads from this forward; None when nothing reads them
|
|
features: "FeatureSink | None" = field(default=None, init=False)
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.rows_per_req is None and self.phase == "decode":
|
|
self.rows_per_req = 1
|
|
# without it, input_mapping[0][::None] would read every row as its own request
|
|
assert self.phase != "verify" or self.rows_per_req is not None, "a verify batch needs rows_per_req"
|
|
|
|
@property
|
|
def is_prefill(self) -> bool:
|
|
return self.phase == "prefill"
|
|
|
|
@property
|
|
def is_decode(self) -> bool:
|
|
return self.phase == "decode"
|
|
|
|
def get_attn_positions(self) -> torch.Tensor:
|
|
return self.mrope_positions if self.mrope_positions is not None else self.positions
|
|
|
|
@property
|
|
def size(self) -> int:
|
|
return len(self.reqs)
|
|
|
|
@property
|
|
def padded_size(self) -> int:
|
|
return len(self.padded_reqs)
|
|
|
|
|
|
@dataclass
|
|
class Context:
|
|
page_size: int
|
|
# NOTE: this table always treat page_size = 1
|
|
page_table: torch.Tensor = field(init=False)
|
|
attn_backend: BaseAttnBackend = field(init=False)
|
|
kv_cache: BaseKVCachePool = field(init=False)
|
|
# Per-request recurrent state for GatedDeltaNet layers; set by the engine for
|
|
# hybrid linear-attention models, otherwise None.
|
|
linear_state_pool: LinearStatePool | None = None
|
|
_batch: Batch | None = field(default=None, init=False)
|
|
|
|
@property
|
|
def batch(self) -> Batch:
|
|
assert self._batch is not None, "No active batch in context"
|
|
return self._batch
|
|
|
|
@contextmanager
|
|
def override(
|
|
self,
|
|
*,
|
|
kv_cache: BaseKVCachePool | None = None,
|
|
attn_backend: BaseAttnBackend | None = None,
|
|
hide_target_state: bool = False,
|
|
):
|
|
"""Swap the KV pool and attention backend for a draft forward; hide_target_state also hides the target's GDN pool."""
|
|
swaps = {}
|
|
if kv_cache is not None:
|
|
swaps["kv_cache"] = kv_cache
|
|
if attn_backend is not None:
|
|
swaps["attn_backend"] = attn_backend
|
|
if hide_target_state:
|
|
swaps["linear_state_pool"] = None
|
|
saved = {name: getattr(self, name) for name in swaps}
|
|
try:
|
|
for name, value in swaps.items():
|
|
setattr(self, name, value)
|
|
yield
|
|
finally:
|
|
for name, value in saved.items():
|
|
setattr(self, name, value)
|
|
|
|
@contextmanager
|
|
def forward_batch(self, batch: Batch, **override):
|
|
assert self._batch is None, "Nested forward_batch is not allowed"
|
|
try:
|
|
self._batch = batch
|
|
with self.override(**override):
|
|
yield
|
|
finally:
|
|
self._batch = None
|
|
|
|
|
|
_GLOBAL_CTX: Context | None = None
|
|
|
|
|
|
def set_global_ctx(ctx: Context):
|
|
global _GLOBAL_CTX
|
|
assert _GLOBAL_CTX is None, "Global context is already set"
|
|
_GLOBAL_CTX = ctx
|
|
|
|
|
|
def get_global_ctx() -> Context:
|
|
assert _GLOBAL_CTX is not None, "Global context is not set"
|
|
return _GLOBAL_CTX
|