mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 16:30:54 +08:00
* feat(mm): serve image input on the Qwen VL families - frontend: OpenAI image_url, Anthropic image blocks and Responses input_image (http(s), base64, gated file://) become MMItems in the tokenizer workers; image spans carry content pseudo ids so the radix cache keys them by content; mrope positions are precomputed per request - registry: EncoderSpec per family (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next register a vision encoder serving image) and ModelSpec.mm_processor; EngineConfig.active_encoders decides what a process builds, served_modalities what it accepts; --text-model-only, --mm-disable, --mm-encoder-weights, --mm-embed-cache-device, --mm-max-pixels - engine and scheduler: per-request encoder-cache row ownership, chunk planning on the batch (jobs, gather rows, scatter rows), items encoded right before the LM forward and gathered into batch.mm_embeds - models: the Qwen VL vision tower on the text tower's TP layers with DeepStack, block weights streamed from pinned host banks by default, merger outputs parked on the host between blocks; SupportsMultimodal (encode, place_encoder_weights) and a shared embed_input_ids - server: image inputs rejected with a client-facing reason when no vision encoder is served; GET /v1/stats reports model.input_modalities * feat(mm): replace the pixel cap with a token budget and processor kwargs - --image-min-tokens / --image-max-tokens replace --mm-max-pixels; MMProcessor.get_mm_processor_kwargs turns the MultimodalConfig into the processor call's arguments, the default passes mm.processor_kwargs through and Qwen VL converts the budget to pixel areas in size.shortest_edge / longest_edge - --mm-processor-kwargs hands a JSON object to the checkpoint's image processor call, applied after the token budget - MMProcessor is an ABC that owns the model path, the config and the lazily loaded image processor; get_mm_processor returns None when --mm-disable leaves no encoder to serve - the frontend tokenizer, the tokenizer worker and the offline LLM build one processor each and hand it to TokenizeManager, so the tokenize message carries no per-request pixel cap
233 lines
9.6 KiB
Python
233 lines
9.6 KiB
Python
from __future__ import annotations
|
|
|
|
import gc
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Dict, List
|
|
|
|
import torch
|
|
from freetoken.core import Batch, Req, get_global_ctx
|
|
from freetoken.distributed import get_tp_info
|
|
from freetoken.utils import init_logger, mem_GB
|
|
from freetoken.utils.progress import emit_progress
|
|
from tqdm import tqdm
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.attention import BaseAttnBackend
|
|
from freetoken.models import BaseLLMModel
|
|
from freetoken.moe.offload_cache import OffloadMoeCache
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class GraphCaptureBuffer:
|
|
input_ids: torch.Tensor
|
|
out_loc: torch.Tensor
|
|
positions: torch.Tensor
|
|
# [3, bs] t/h/w rope positions; allocated only for mrope models (else None).
|
|
mrope_positions: torch.Tensor | None
|
|
logits: torch.Tensor
|
|
table_idx: torch.Tensor # per-request slot id for GatedDeltaNet state gather/scatter
|
|
# Decode GDN query indptr = arange(bs+1); a constant per captured bs, filled once.
|
|
fla_cu_seqlens: torch.Tensor
|
|
|
|
@classmethod
|
|
def init(
|
|
cls, bs: int, vocab_size: int, device: torch.device, mrope: bool = False
|
|
) -> GraphCaptureBuffer:
|
|
return GraphCaptureBuffer(
|
|
input_ids=torch.zeros(bs, dtype=torch.int32, device=device),
|
|
out_loc=torch.zeros(bs, dtype=torch.int32, device=device),
|
|
positions=torch.zeros(bs, dtype=torch.int32, device=device),
|
|
mrope_positions=(
|
|
torch.zeros(3, bs, dtype=torch.int32, device=device) if mrope else None
|
|
),
|
|
logits=torch.empty(bs, vocab_size, dtype=torch.float32, device=device),
|
|
table_idx=torch.zeros(bs, dtype=torch.int32, device=device),
|
|
fla_cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device),
|
|
)
|
|
|
|
def set_batch(self, batch: Batch) -> None:
|
|
from freetoken.attention.linear import FLAMetadata
|
|
|
|
_slice = slice(batch.padded_size)
|
|
bs = batch.padded_size
|
|
batch.input_ids = self.input_ids[_slice]
|
|
batch.out_loc = self.out_loc[_slice]
|
|
batch.positions = self.positions[_slice]
|
|
if self.mrope_positions is not None:
|
|
batch.mrope_positions = self.mrope_positions[:, _slice]
|
|
batch.linear_table_idx = self.table_idx[_slice]
|
|
# Decode GDN metadata reads the persistent cu_seqlens (constant arange) and the
|
|
# persistent table_idx slot map, so the captured kernels see stable addresses.
|
|
batch.fla_metadata = FLAMetadata(
|
|
cu_seqlens=self.fla_cu_seqlens[: bs + 1], cache_indices=self.table_idx[_slice]
|
|
)
|
|
|
|
def copy_from(self, batch: Batch) -> None:
|
|
_slice = slice(batch.padded_size)
|
|
self.input_ids[_slice] = batch.input_ids
|
|
if batch.out_loc is not None:
|
|
self.out_loc[_slice] = batch.out_loc
|
|
self.positions[_slice] = batch.positions
|
|
if self.mrope_positions is not None:
|
|
self.mrope_positions[:, _slice] = batch.mrope_positions
|
|
if batch.linear_table_idx is not None:
|
|
self.table_idx[_slice] = batch.linear_table_idx
|
|
|
|
|
|
def _determine_cuda_graph_bs(
|
|
cuda_graph_bs: List[int] | None,
|
|
cuda_graph_max_bs: int | None,
|
|
free_memory: int,
|
|
) -> List[int]:
|
|
if cuda_graph_bs is not None:
|
|
return cuda_graph_bs
|
|
|
|
free_memory_gb = free_memory / (1 << 30)
|
|
if cuda_graph_max_bs is None:
|
|
if free_memory_gb > 80: # H200
|
|
cuda_graph_max_bs = 256
|
|
else:
|
|
cuda_graph_max_bs = 160
|
|
|
|
if cuda_graph_max_bs < 1:
|
|
return []
|
|
|
|
candidates = [1, 2, 4] + list(range(8, cuda_graph_max_bs + 1, 8))
|
|
return [bs for bs in candidates if bs <= cuda_graph_max_bs]
|
|
|
|
|
|
def get_free_memory(device: torch.device) -> int:
|
|
return torch.cuda.mem_get_info(device)[0]
|
|
|
|
|
|
class GraphRunner:
|
|
def __init__(
|
|
self,
|
|
stream: torch.cuda.Stream,
|
|
device: torch.device,
|
|
model: BaseLLMModel,
|
|
attn_backend: BaseAttnBackend,
|
|
cuda_graph_bs: List[int] | None,
|
|
cuda_graph_max_bs: int | None,
|
|
free_memory: int,
|
|
max_seq_len: int,
|
|
vocab_size: int,
|
|
dummy_req: Req,
|
|
moe_offload_cache: OffloadMoeCache | None = None,
|
|
mrope: bool = False,
|
|
) -> None:
|
|
cuda_graph_bs = _determine_cuda_graph_bs(
|
|
cuda_graph_bs=cuda_graph_bs,
|
|
cuda_graph_max_bs=cuda_graph_max_bs,
|
|
free_memory=free_memory,
|
|
)
|
|
self.attn_backend = attn_backend
|
|
self.max_graph_bs = max(cuda_graph_bs) if cuda_graph_bs else 0
|
|
self.graph_bs_list = sorted(cuda_graph_bs)
|
|
self.dummy_req = dummy_req
|
|
self.moe_offload_cache = moe_offload_cache
|
|
self.mrope = mrope
|
|
self.stream = stream
|
|
self.device = device
|
|
self._capture_graphs(max_seq_len, vocab_size, model)
|
|
|
|
def _reset_moe_offload_cache(self) -> None:
|
|
if self.moe_offload_cache is not None:
|
|
self.moe_offload_cache.reset()
|
|
|
|
def _capture_graphs(self, max_seq_len: int, vocab_size: int, model: BaseLLMModel):
|
|
# Mark the post-weights "warmup" phase for /health: this stretch (graph capture — or the
|
|
# remaining readiness work when graphs are disabled) moves no bytes, so without this the
|
|
# loader would sit at 100% (last byte bar) until the ready ack. total=0 ⇒ the desktop
|
|
# reads it as an indeterminate phase and animates the bar. Must precede the
|
|
# graphs-disabled early return so that config gets the phase too.
|
|
emit_progress("Capturing CUDA graphs / warming up", 0, 0)
|
|
self.graph_map: Dict[int, torch.cuda.CUDAGraph] = {}
|
|
if self.max_graph_bs == 0:
|
|
return logger.info_rank0("CUDA graph is disabled.")
|
|
|
|
self.attn_backend.init_capture_graph(max_seq_len=max_seq_len, bs_list=self.graph_bs_list)
|
|
|
|
torch.cuda.synchronize(self.device)
|
|
torch.cuda.empty_cache()
|
|
torch.cuda.reset_peak_memory_stats(self.device)
|
|
|
|
logger.info_rank0(f"Start capturing CUDA graphs with sizes: {self.graph_bs_list}")
|
|
free_memory = get_free_memory(self.device)
|
|
logger.info_rank0(f"Free GPU memory before capturing CUDA graphs: {mem_GB(free_memory)}")
|
|
|
|
self.buffer = GraphCaptureBuffer.init(
|
|
self.max_graph_bs, vocab_size, self.device, mrope=self.mrope
|
|
)
|
|
self._reset_moe_offload_cache()
|
|
|
|
pbar = tqdm(
|
|
sorted(self.graph_bs_list, reverse=True),
|
|
desc="Preparing for capturing CUDA graphs...",
|
|
unit="batch",
|
|
disable=not get_tp_info().is_primary(), # disable for non-primary ranks
|
|
)
|
|
pool = None
|
|
for bs in pbar:
|
|
free_memory = get_free_memory(self.device)
|
|
pbar.desc = f"Capturing graphs: bs = {bs:<3} | avail_mem = {mem_GB(free_memory)}"
|
|
pbar.refresh()
|
|
graph = torch.cuda.CUDAGraph()
|
|
batch = Batch(reqs=[self.dummy_req] * bs, phase="decode")
|
|
batch.padded_reqs = batch.reqs
|
|
self.attn_backend.prepare_for_capture(batch)
|
|
self.buffer.set_batch(batch)
|
|
# capture on the dummy linear-state slot so GatedDeltaNet gather/scatter
|
|
# touches scratch (real slot indices are written by copy_from on replay). Hybrid-
|
|
# radix decouples the GDN slot from table_idx -> use the GDN padding slot.
|
|
dummy_slot = (self.dummy_req.linear_slot_idx
|
|
if self.dummy_req.linear_slot_idx is not None
|
|
else self.dummy_req.table_idx)
|
|
self.buffer.table_idx[:bs].fill_(dummy_slot)
|
|
with get_global_ctx().forward_batch(batch):
|
|
self.buffer.logits[:bs] = model.forward()
|
|
# Keep the offload cache warmed for capture. Resetting here forces
|
|
# CUDA graph capture to replay cold-cache expert copies.
|
|
with torch.cuda.graph(graph, pool=pool, stream=self.stream):
|
|
self.buffer.logits[:bs] = model.forward()
|
|
self._reset_moe_offload_cache()
|
|
if pool is None:
|
|
pool = graph.pool() # reuse cuda graph handle to reduce memory
|
|
self.graph_map[bs] = graph
|
|
|
|
self._reset_moe_offload_cache()
|
|
free_memory = get_free_memory(self.device)
|
|
logger.info_rank0(f"Free GPU memory after capturing CUDA graphs: {mem_GB(free_memory)}")
|
|
|
|
def can_use_cuda_graph(self, batch: Batch) -> bool:
|
|
return batch.is_decode and batch.size <= self.max_graph_bs
|
|
|
|
def replay(self, batch: Batch) -> torch.Tensor:
|
|
assert self.can_use_cuda_graph(batch)
|
|
self.buffer.copy_from(batch)
|
|
g = self.graph_map[batch.padded_size]
|
|
self.attn_backend.prepare_for_replay(batch)
|
|
g.replay()
|
|
return self.buffer.logits[: batch.size]
|
|
|
|
def pad_batch(self, batch: Batch) -> None:
|
|
padded_size = ( # choose the first available batch size
|
|
next(bs for bs in self.graph_bs_list if bs >= batch.size)
|
|
if self.can_use_cuda_graph(batch)
|
|
else batch.size
|
|
)
|
|
batch.padded_reqs = batch.reqs + [self.dummy_req] * (padded_size - batch.size)
|
|
|
|
# NOTE: This must be called before freeing NCCL resources to prevent program hang
|
|
def destroy_cuda_graphs(self) -> None:
|
|
# Drop the CUDAGraph objects (and the shared mempool they hold) AND the static
|
|
# GraphCaptureBuffer tensors ([max_bs, vocab] logits + input/out_loc/positions/...).
|
|
# Dropping the references is the load-bearing step; without it a runtime rebuild's
|
|
# free-before-alloc cannot reclaim this GPU memory. empty_cache() is left to the
|
|
# caller / next capture (GraphRunner._capture_graphs already runs it).
|
|
self.graph_map = {}
|
|
self.buffer = None
|
|
gc.collect()
|