mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor: add engine and scheduler seams for multi-row decode steps * refactor: rename the graph steps argument to rows_per_req * refactor: drop VerifyInfo and require rows_per_req on verify batches * refactor: fix stale seam comments and read decode_rows_per_req directly * refactor: reword two GraphRunner comments for multi-row graphs * fix(engine): scale the GPU-decode MoE pool limits by decode_rows_per_req * refactor: drop the unread Context.moe_offload_cache field * test: trim the multi-row seam tests
25 lines
954 B
Python
25 lines
954 B
Python
from dataclasses import dataclass
|
|
|
|
import torch
|
|
|
|
|
|
@dataclass
|
|
class BaseCaptureData:
|
|
seq_lens: torch.Tensor
|
|
positions: torch.Tensor
|
|
cu_seqlens_k: torch.Tensor
|
|
cu_seqlens_q: torch.Tensor
|
|
page_table: torch.Tensor
|
|
|
|
@classmethod
|
|
def create(cls, max_bs: int, max_seq_len: int, device: torch.device, rows_per_req: int = 1, **kwargs):
|
|
return cls(
|
|
seq_lens=torch.ones((max_bs,), dtype=torch.int32, device=device),
|
|
positions=torch.zeros((max_bs * rows_per_req,), dtype=torch.int32, device=device),
|
|
cu_seqlens_k=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device),
|
|
# constant under a graph: every request has exactly `rows_per_req` query rows
|
|
cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device) * rows_per_req,
|
|
page_table=torch.zeros((max_bs, max_seq_len), dtype=torch.int32, device=device),
|
|
**kwargs,
|
|
)
|