Files
FreeToken/python/freetoken/engine/spec_protocol.py
Xiaoze Fan 96dbe1dd14 feat(spec): add --speculative-token-map for a smaller mtp draft head (#892)
* feat(spec): add --speculative-token-map for a smaller mtp draft head

* feat(spec): add scripts/make_token_map.py to write a token map

* fix(spec): correct the token map vram note and seed its test
2026-11-11 12:15:09 -08:00

57 lines
1.7 KiB
Python

"""The speculative-decoding runtime as the engine and scheduler see it; they import these types only, never ``freetoken.spec``."""
from __future__ import annotations
from typing import TYPE_CHECKING, List, NamedTuple, Protocol
import torch
if TYPE_CHECKING:
from freetoken.core import Batch, Req
from freetoken.engine.sample import BatchSamplingArgs
from freetoken.models.blocks import FeatureSink
from freetoken.models.config import SlotStateSpec
SPEC_RUNNER = "freetoken.spec.runner:SpecRunner"
class SpecForwardOutput(NamedTuple):
"""Drop-in for ForwardOutput at the drain: row i committed ``next_tokens[i, : emitted[i] + 1]``."""
next_tokens_gpu: torch.Tensor # [bs, k+1]
next_tokens_cpu: torch.Tensor
copy_done_event: torch.cuda.Event
emitted_cpu: torch.Tensor # [bs] accepted drafts, 0..k
class SpecRuntime(Protocol):
radix_lookahead: bool
@classmethod
def adjust_config(cls, config, override) -> None: ...
@classmethod
def prepare_target(cls, config, model) -> None: ...
@classmethod
def budget(cls, config) -> tuple[int, int]: ...
@classmethod
def slot_states(cls, config) -> tuple[SlotStateSpec, ...]: ...
def attach(self, engine) -> None: ...
def detach(self, *, keep_kv: bool) -> None: ...
def capture_graphs(self, bs_list: List[int], pool) -> None: ...
def feature_sink(self, batch: Batch) -> FeatureSink | None: ...
def on_prefill(self, batch: Batch, next_tokens_gpu: torch.Tensor) -> None: ...
def propose_into(self, reqs: List[Req], token_pool: torch.Tensor) -> None: ...
def verify_step(self, batch: Batch, sample_args: BatchSamplingArgs) -> SpecForwardOutput: ...
def release(self, req: Req) -> None: ...