Files

359 lines
14 KiB
Python

from __future__ import annotations
import functools
import math
from typing import Any, Mapping, Tuple
import torch
from .base import StateLessOP
class RotaryEmbedding(StateLessOP):
def __init__(
self,
head_size: int,
rotary_dim: int,
cos_sin_cache: torch.Tensor,
is_neox: bool = True,
) -> None:
super().__init__()
assert 0 < rotary_dim <= head_size
assert rotary_dim % 2 == 0
assert head_size in [64, 128, 256, 512]
self.head_size = head_size
self.rotary_dim = rotary_dim
# NeoX (half-rotation, HF default) vs GPT-J interleaved (adjacent pairs,
# ``rope_interleave`` models: GLM MLA lineage). Both underlying kernels
# accept the flag; the cos/sin cache layout is identical.
self.is_neox = is_neox
# shared with every rope of the same frequencies (rope_cos_sin_table), so never written
self._cos_sin_cache = cos_sin_cache
from freetoken.kernel.backend import is_flashinfer_installed
if is_flashinfer_installed():
from flashinfer import apply_rope_with_cos_sin_cache_inplace
else:
from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace
self.apply_rope_with_cos_sin_cache_inplace = apply_rope_with_cos_sin_cache_inplace
def forward(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
self.apply_rope_with_cos_sin_cache_inplace(
positions=positions,
query=query,
key=key,
head_size=self.head_size,
cos_sin_cache=self._cos_sin_cache,
is_neox=self.is_neox,
)
return query, key
def mrope_cos_sin_rows(
cos_sin_cache: torch.Tensor, positions: torch.Tensor, section_table: torch.Tensor
) -> torch.Tensor:
"""Per-token [cos | sin] rows for [3, n] positions: frequency slot i reads the cache row of axis section_table[i]."""
half = cos_sin_cache.shape[1] // 2
pos = positions[section_table.long(), :].transpose(0, 1).long() # [n, half]
idx = torch.arange(half, device=cos_sin_cache.device)
return torch.cat((cos_sin_cache[pos, idx], cos_sin_cache[pos, half + idx]), dim=1)
def _mrope_torch(
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
head_size: int,
cos_sin_cache: torch.Tensor,
section_table: torch.Tensor,
) -> None:
"""Pure-torch mrope for triton-less installs; same in-place NeoX contract as the kernel."""
nnz = query.shape[0]
if nnz == 0:
return
rotary_dim = cos_sin_cache.shape[1]
half = rotary_dim // 2
rows = mrope_cos_sin_rows(cos_sin_cache, positions, section_table)
cos = rows[:, :half].unsqueeze(1).float()
sin = rows[:, half:].unsqueeze(1).float()
for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):
v = t.view(nnz, heads, head_size)
lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()
# Compute both halves before writing: float() aliases FP32 inputs.
lo_new = lo * cos - hi * sin
hi_new = hi * cos + lo * sin
v[..., :half] = lo_new.to(v.dtype)
v[..., half:rotary_dim] = hi_new.to(v.dtype)
MROPE_LAYOUTS = ("contiguous", "interleaved", "interleaved_glm")
def build_section_table(mrope_section: tuple[int, int, int], layout: str) -> torch.Tensor:
"""Axis id (0=t, 1=h, 2=w) per frequency slot for a [t, h, w] section split.
contiguous: T|H|W blocks. interleaved: H at slots 1,4,.. and W at 2,5,.. until each share is used, T takes the rest.
interleaved_glm: round-robin over the three axes, skipping an axis once its share is used up.
"""
st, sh, sw = mrope_section
half = st + sh + sw
sec = torch.zeros(half, dtype=torch.int32)
if layout == "contiguous":
sec[st : st + sh] = 1
sec[st + sh :] = 2
elif layout == "interleaved":
sec[1 : 3 * sh : 3] = 1
sec[2 : 3 * sw : 3] = 2
if int((sec == 1).sum()) != sh or int((sec == 2).sum()) != sw:
raise ValueError(f"mrope_section {mrope_section} is not representable in the interleaved layout")
elif layout == "interleaved_glm":
counts = [0, 0, 0]
for i in range(half):
ax = i % 3
while counts[ax] >= mrope_section[ax]:
ax = (ax + 1) % 3
sec[i] = ax
counts[ax] += 1
else:
raise ValueError(f"unknown mrope layout {layout!r}; expected one of {MROPE_LAYOUTS}")
return sec
class MRotaryEmbedding(RotaryEmbedding):
"""3-axis (t/h/w) rope over the parent's cos_sin_cache; section_table picks the axis row per frequency slot. Consumes positions [3, n]."""
def __init__(
self, *args, mrope_section: tuple, layout: str = "interleaved", **kwargs
) -> None:
super().__init__(*args, **kwargs)
assert self.is_neox, "mrope is defined on the NeoX half-rotation layout"
assert self._cos_sin_cache.shape[1] == self.rotary_dim, "mrope reads a rotary_dim-wide table, not a proportional rope's"
half = self.rotary_dim // 2
assert sum(mrope_section) == half, (mrope_section, half)
self._section_table = build_section_table(tuple(mrope_section), layout)
try:
from freetoken.kernel.triton.rope import apply_mrope_with_cos_sin_cache_inplace
self._kernel = apply_mrope_with_cos_sin_cache_inplace
except ImportError:
self._kernel = None
def forward(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
assert positions.dim() == 2, "mrope models feed [3, n] positions"
if self._section_table.device != query.device:
self._section_table = self._section_table.to(query.device)
positions = positions.contiguous()
if self._kernel is not None:
self._kernel(
positions=positions, query=query, key=key, head_size=self.head_size,
cos_sin_cache=self._cos_sin_cache, section_table=self._section_table,
)
else:
_mrope_torch(
positions, query, key, self.head_size,
self._cos_sin_cache, self._section_table,
)
return query, key
def rope_table_positions(max_position: int, rope_scaling: Mapping[str, Any] | None) -> int:
"""Rows the cos/sin table needs: YaRN serves ``original_max_position_embeddings * factor``,
above ``max_position_embeddings`` when only the rope parameters are overridden."""
if rope_scaling is None or rope_scaling.get("rope_type") != "yarn":
return max_position
extended = int(rope_scaling["original_max_position_embeddings"] * rope_scaling["factor"])
return max(max_position, extended)
def _rope_frequencies(
head_size: int, rotary_dim: int, base: float, rope_scaling: Mapping[str, Any] | None
) -> Tuple[torch.Tensor, float]:
"""``inv_freq`` over the table's half width, and the factor cos and sin are both scaled by."""
rope_type = "default" if rope_scaling is None else rope_scaling["rope_type"]
if rope_type == "proportional":
# Frequencies spaced over the whole head; the pairs past rotary_dim do not rotate.
inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2, dtype=torch.float) / head_size))
inv_freq[rotary_dim // 2 :] = 0.0
return inv_freq, 1.0
# Standard (NeoX) rope. Supports partial rotary (rotary_dim < head_size): rope is applied
# to the first ``rotary_dim`` dims of each head, the rest pass through, and frequencies
# are spaced over ``rotary_dim`` (HF default partial rope, e.g. Qwen3.5
# partial_rotary_factor, MiniMax-M2's ``apply_rotary_pos_emb``).
inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
match rope_type:
case "default":
return inv_freq, 1.0
case "llama3":
scaling_factor: float = rope_scaling["factor"]
low_freq_factor: float = rope_scaling["low_freq_factor"]
high_freq_factor: float = rope_scaling["high_freq_factor"]
original_max_position: int = rope_scaling["original_max_position_embeddings"]
wave_len = 2 * math.pi / inv_freq
# no smooth if low_freq_factor == high_freq_factor
if low_freq_factor == high_freq_factor:
return torch.where(
wave_len < original_max_position / high_freq_factor,
inv_freq,
inv_freq / scaling_factor,
), 1.0
delta = high_freq_factor - low_freq_factor
smooth = torch.clamp((original_max_position / wave_len - low_freq_factor) / delta, 0, 1)
return ((1 - smooth) / scaling_factor + smooth) * inv_freq, 1.0
case "yarn":
factor: float = rope_scaling["factor"]
beta_fast: float = rope_scaling.get("beta_fast", 32.0)
beta_slow: float = rope_scaling.get("beta_slow", 1.0)
orig_max_pos: int = rope_scaling["original_max_position_embeddings"]
def get_mscale(scale: float, mscale: float = 1.0) -> float:
if scale <= 1:
return 1.0
return 0.1 * mscale * math.log(scale) + 1.0
attention_factor = rope_scaling.get("attention_factor")
if attention_factor is None:
mscale = rope_scaling.get("mscale")
mscale_all_dim = rope_scaling.get("mscale_all_dim")
# Truthiness, not presence: HF falls back to get_mscale(factor) when
# mscale_all_dim is 0 (a real DeepSeek-lineage default).
if mscale and mscale_all_dim:
attention_factor = get_mscale(factor, mscale) / get_mscale(
factor, mscale_all_dim
)
else:
attention_factor = get_mscale(factor)
def _find_correction_dim(num_rotations: float) -> float:
return (
rotary_dim
* math.log(orig_max_pos / (num_rotations * 2 * math.pi))
/ (2 * math.log(base))
)
low = _find_correction_dim(beta_fast)
high = _find_correction_dim(beta_slow)
if rope_scaling.get("truncate", True):
low = math.floor(low)
high = math.ceil(high)
low = max(low, 0)
# rotary_dim - 1, per HF's find_correction_range and this repo's own faithful copy in
# models/deepseek_v4/ops.py. Clamping to rotary_dim//2 - 1 instead forces the ramp to
# reach 1.0 at the last entry, fully interpolating the longest-wavelength dims that
# the reference deliberately leaves partly extrapolated.
high = min(high, rotary_dim - 1)
if low == high: # HF nudges instead of flooring the gap at 1 ("truncate": false)
high += 0.001
ramp = torch.clamp(
(torch.arange(rotary_dim // 2, dtype=torch.float32) - low) / (high - low), 0, 1
)
return (inv_freq / factor) * ramp + inv_freq * (1 - ramp), float(attention_factor)
raise ValueError(f"Unsupported {rope_scaling = }")
@functools.cache
def _cos_sin_table(
head_size: int,
rotary_dim: int,
positions: int,
base: float,
rope_scaling: Tuple[Tuple[str, Any], ...] | None,
device: torch.device,
) -> torch.Tensor:
inv_freq, attention_factor = _rope_frequencies(
head_size, rotary_dim, base, dict(rope_scaling) if rope_scaling is not None else None
)
freqs = torch.arange(positions, dtype=torch.float)[:, None] * inv_freq[None, :]
return torch.cat((freqs.cos() * attention_factor, freqs.sin() * attention_factor), dim=-1)
def rope_cos_sin_table(
head_size: int,
rotary_dim: int,
max_position: int,
base: float,
rope_scaling: Tuple[Tuple[str, Any], ...] | None = None,
) -> torch.Tensor:
"""The ``[positions, width]`` cos|sin table, one per distinct rope: only proportional rope
depends on the head size, so an indexer whose heads differ in width reads the attention's table."""
rope_map = dict(rope_scaling) if rope_scaling is not None else None
proportional = rope_map is not None and rope_map["rope_type"] == "proportional"
device = torch.get_default_device()
if device.type == "cuda" and device.index is None:
device = torch.device("cuda", torch.cuda.current_device())
return _cos_sin_table(
head_size if proportional else rotary_dim,
rotary_dim,
rope_table_positions(max_position, rope_map),
base,
rope_scaling,
device,
)
_ROPE_DEVICE: torch.device | None = None
def set_rope_device(device: torch.device):
global _ROPE_DEVICE
_ROPE_DEVICE = device
@functools.cache
def get_rope(
head_dim: int,
rotary_dim: int,
max_position: int,
base: float,
rope_scaling: Tuple[Tuple[str, Any], ...] | None = None,
is_neox: bool = True,
mrope_section: Tuple[int, ...] | None = None,
mrope_layout: str = "interleaved",
) -> RotaryEmbedding:
def build() -> RotaryEmbedding:
table = rope_cos_sin_table(head_dim, rotary_dim, max_position, base, rope_scaling)
if mrope_section is not None:
return MRotaryEmbedding(
head_dim, rotary_dim, table,
is_neox=is_neox, mrope_section=tuple(mrope_section),
layout=mrope_layout,
)
return RotaryEmbedding(head_dim, rotary_dim, table, is_neox=is_neox)
t = torch.tensor([])
if t.device == torch.device("meta"):
# we cannot use meta device for rope
if _ROPE_DEVICE is None:
raise RuntimeError(
"We cannot use meta device for rope. Please call set_rope_device() first."
)
with torch.device(_ROPE_DEVICE):
return build()
return build()
__all__ = [
"MROPE_LAYOUTS",
"MRotaryEmbedding",
"RotaryEmbedding",
"build_section_table",
"get_rope",
"rope_cos_sin_table",
"rope_table_positions",
"set_rope_device",
]