mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
92 lines
4.1 KiB
Python
92 lines
4.1 KiB
Python
"""Scheduler-side multimodal planning: the embedding rows a chunk gathers and the items it encodes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, List, Tuple
|
|
|
|
from freetoken.utils import align_down
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.message import MMItem
|
|
from freetoken.mm.encoder_cache import EncoderCache
|
|
|
|
|
|
def mm_rows_after(item: MMItem, cached_len: int) -> int:
|
|
"""Embedding rows of the item that prefill will gather once the first cached_len tokens are skipped."""
|
|
return sum(max(0, span_hi - max(span_lo, cached_len)) for span_lo, span_hi in item.offsets)
|
|
|
|
|
|
def mm_chunk_end(items: List[MMItem], chunk_lo: int, end: int, align: int) -> int:
|
|
"""The chunk end pulled back to an align multiple before any image span it would cut; end itself when that span already begins at or before chunk_lo (the image is longer than the budget)."""
|
|
cut = end
|
|
for span_lo, span_hi in sorted((lo, hi) for item in items for lo, hi in item.offsets)[::-1]:
|
|
if span_lo < cut < span_hi:
|
|
if span_lo <= chunk_lo:
|
|
return end
|
|
cut = align_down(span_lo, align)
|
|
if cut <= chunk_lo:
|
|
return end
|
|
return cut
|
|
|
|
|
|
def cut_image_spans(reqs) -> List[Tuple[int, int]]:
|
|
"""The image spans the reqs' current chunks end inside of; mm_chunk_end leaves such a cut only for an image longer than the chunk."""
|
|
return [(lo, hi) for req in reqs if req.mm_items for item in req.mm_items for lo, hi in item.offsets if lo < req.device_len < hi]
|
|
|
|
|
|
def plan_mm_chunk(
|
|
uid: int,
|
|
items: List[MMItem],
|
|
window_lo: int,
|
|
window_hi: int,
|
|
encoder_cache: EncoderCache | None,
|
|
) -> tuple[List[MMItem], List[Tuple[int, int, int, int, int, int]]]:
|
|
"""Encoder jobs and the embedding-row gather plan for the items overlapping the chunk window [window_lo, window_hi).
|
|
|
|
Plan rows are (uid, hash, row_lo, row_hi, num_tokens, pos): embedding rows [row_lo, row_hi) land at chunk position pos. A repeated image is one job.
|
|
"""
|
|
jobs: List[MMItem] = []
|
|
queued: set[int] = set()
|
|
plan: List[Tuple[int, int, int, int, int, int]] = []
|
|
for item in items:
|
|
num_tokens = item.num_tokens
|
|
needs_encode = item.hash not in queued and (encoder_cache is None or not encoder_cache.has(item.hash))
|
|
row_base = 0
|
|
for span_lo, span_hi in item.offsets:
|
|
lo, hi = max(span_lo, window_lo), min(span_hi, window_hi)
|
|
if lo < hi:
|
|
if needs_encode:
|
|
jobs.append(item)
|
|
queued.add(item.hash)
|
|
needs_encode = False
|
|
plan.append((uid, item.hash, row_base + lo - span_lo, row_base + hi - span_lo, num_tokens, lo - window_lo))
|
|
row_base += span_hi - span_lo
|
|
return jobs, plan
|
|
|
|
|
|
def plan_mm_batch(reqs, encoder_cache: EncoderCache | None) -> tuple[List[MMItem], List[Tuple[int, int, int, int, int, int]], List[int], List[int]]:
|
|
"""Jobs, plan, the batch rows of every gathered embedding row and, per batch token, the end of the image span holding it (0 for text), over the reqs in batch order (each spans [cached_len, device_len))."""
|
|
jobs: List[MMItem] = []
|
|
plan: List[Tuple[int, int, int, int, int, int]] = []
|
|
rows: List[int] = []
|
|
block_ends: List[int] = []
|
|
offset = 0
|
|
for req in reqs:
|
|
if req.mm_items:
|
|
req_jobs, req_plan = plan_mm_chunk(req.uid, req.mm_items, req.cached_len, req.device_len, encoder_cache)
|
|
jobs.extend(req_jobs)
|
|
plan.extend(req_plan)
|
|
for _, _, row_lo, row_hi, _, pos in req_plan:
|
|
rows.extend(range(offset + pos, offset + pos + row_hi - row_lo))
|
|
if not block_ends:
|
|
block_ends = [0] * sum(r.extend_len for r in reqs)
|
|
for item in req.mm_items:
|
|
for span_lo, span_hi in item.offsets:
|
|
for i in range(max(span_lo, req.cached_len), min(span_hi, req.device_len)):
|
|
block_ends[offset + i - req.cached_len] = span_hi
|
|
offset += req.extend_len
|
|
return jobs, plan, rows, block_ends
|
|
|
|
|
|
__all__ = ["cut_image_spans", "mm_chunk_end", "mm_rows_after", "plan_mm_batch", "plan_mm_chunk"]
|