mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): add the speculative verify infrastructure * feat(scheduler): run decode steps as speculative verify steps * chore(engine): mark the speculative decoding config fields with one comment * feat(server): add short aliases for the speculative decoding flags
50 lines
1.5 KiB
Python
50 lines
1.5 KiB
Python
"""The speculative-decoding runtime as the engine and scheduler see it; they import these types only, never ``freetoken.spec``."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, List, NamedTuple, Protocol
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.core import Batch, Req
|
|
from freetoken.engine.sample import BatchSamplingArgs
|
|
from freetoken.models.blocks import FeatureSink
|
|
|
|
SPEC_RUNNER = "freetoken.spec.runner:SpecRunner"
|
|
|
|
|
|
class SpecForwardOutput(NamedTuple):
|
|
"""Drop-in for ForwardOutput at the drain: row i committed ``next_tokens[i, : emitted[i] + 1]``."""
|
|
|
|
next_tokens_gpu: torch.Tensor # [bs, k+1]
|
|
next_tokens_cpu: torch.Tensor
|
|
copy_done_event: torch.cuda.Event
|
|
emitted_cpu: torch.Tensor # [bs] accepted drafts, 0..k
|
|
|
|
|
|
class SpecRuntime(Protocol):
|
|
radix_lookahead: bool
|
|
|
|
@classmethod
|
|
def adjust_config(cls, config, override) -> None: ...
|
|
|
|
@classmethod
|
|
def budget(cls, config) -> tuple[int, int]: ...
|
|
|
|
def attach(self, engine) -> None: ...
|
|
|
|
def detach(self, *, keep_kv: bool) -> None: ...
|
|
|
|
def capture_graphs(self, bs_list: List[int], pool) -> None: ...
|
|
|
|
def feature_sink(self, batch: Batch) -> FeatureSink | None: ...
|
|
|
|
def on_prefill(self, batch: Batch, next_tokens_gpu: torch.Tensor) -> None: ...
|
|
|
|
def propose_into(self, reqs: List[Req], token_pool: torch.Tensor) -> None: ...
|
|
|
|
def verify_step(self, batch: Batch, sample_args: BatchSamplingArgs) -> SpecForwardOutput: ...
|
|
|
|
def release(self, req: Req) -> None: ...
|