Files
Mukkesh Ganesh 58b3ad7ba7 feat(gemma4): speculative decoding with the assistant draft (#895)
* fix(scheduler): return the sliding-window slots when a verify step rolls back

* feat(gemma4): add the assistant draft model and its loader

* feat(spec): serve sliding-window targets with a draft that keeps no KV

* fix(spec): budget verify rows in the swa pool and drop the post-verify sync
2026-11-12 20:34:28 -08:00

60 lines
1.8 KiB
Python

"""The speculative-decoding runtime as the engine and scheduler see it; they import these types only, never ``freetoken.spec``."""
from __future__ import annotations
from typing import TYPE_CHECKING, List, NamedTuple, Protocol
import torch
if TYPE_CHECKING:
from freetoken.core import Batch, Req
from freetoken.engine.sample import BatchSamplingArgs
from freetoken.models.blocks import FeatureSink
from freetoken.models.config import SlotStateSpec
SPEC_RUNNER = "freetoken.spec.runner:SpecRunner"
class SpecForwardOutput(NamedTuple):
"""Drop-in for ForwardOutput at the drain: row i committed ``next_tokens[i, : emitted[i] + 1]``."""
next_tokens_gpu: torch.Tensor # [bs, k+1]
next_tokens_cpu: torch.Tensor
copy_done_event: torch.cuda.Event
emitted_cpu: torch.Tensor # [bs] accepted drafts, 0..k
class SpecRuntime(Protocol):
radix_lookahead: bool
@classmethod
def adjust_config(cls, config, override) -> None: ...
@classmethod
def prepare_target(cls, config, model) -> None: ...
@classmethod
def budget(cls, config) -> tuple[int, int]: ...
@classmethod
def slot_states(cls, config) -> tuple[SlotStateSpec, ...]: ...
@classmethod
def target_may_be_windowed(cls, config) -> bool: ...
def attach(self, engine) -> None: ...
def detach(self, *, keep_kv: bool) -> None: ...
def capture_graphs(self, bs_list: List[int], pool) -> None: ...
def feature_sink(self, batch: Batch) -> FeatureSink | None: ...
def on_prefill(self, batch: Batch, next_tokens_gpu: torch.Tensor) -> None: ...
def propose_into(self, reqs: List[Req], token_pool: torch.Tensor) -> None: ...
def verify_step(self, batch: Batch, sample_args: BatchSamplingArgs) -> SpecForwardOutput: ...
def release(self, req: Req) -> None: ...