Files
Xiaoze Fan bd8f3d519a feat(qwen4_exp): support Qwen3.8-Flash-Next (#257)
Serve Qwen3.8-Flash-Next (HF model_type qwen4_exp) text-only: 36 GDN +
12 QSA compressed-sparse attention layers on 4 hyper-connection residual
streams, a PLE n-gram embedding layer backed by a 47.7 GiB pinned-host
table with UVA gather, and 512 NVFP4 / block-fp8 routed experts (top-10)
plus a gated shared expert.

- attention: qsa_sparse backend (AttnType.QSA) over QSAKVCache -- paged
  GQA K/V, a 1/ratio compressed index-key slab shadowing the KV pages,
  and a per-request pending ring sized from index_ratio
- kvcache: declarative slot-sibling states (ModelConfig.slot_states) on
  LinearStatePool carry the PLE conv history and n-gram context through
  the hybrid-radix snapshot/COW lifecycle
- scheduler: hybrid prefill chunks align to the page size so snapshots
  land on donatable boundaries
- kernels: triton kernels adapted from vLLM/SGLang (hc, qsa, ple gather,
  moe router / shared gate) plus an original radix block top-k; int64
  row addressing throughout
- moe: non-power-of-2 top-k router, deep-K marlin decode config, one
  fp8 scale-bank padding rule shared with the AOT row table
- engine: the PLE table load reserves its pinned bytes from the pin
  budget before the expert banks plan their residency
2026-08-28 15:33:15 -07:00

96 lines
3.2 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
# Adapted from SGLang (python/sglang/srt/models/qwen4_exp.py)
"""UVA row gather for the Qwen3.8-Flash-Next PLE n-gram table.
The table (320,001,536 rows x 160, FP8-e4m3 + one scalar scale = 47.7 GiB) stays in pinned
host memory and the GPU dereferences it in place over PCIe -- at its host VA on Linux/UVA, at
the mapped device address on WDDM (``kernel/pinned.device_ptr``). One program per requested
row: read the row, widen to fp32, apply the per-tensor scale, store bf16.
Ids outside the table store zeros.
"""
from __future__ import annotations
import torch
import triton
import triton.language as tl
from freetoken.kernel.triton.e4m3_compat import e4m3_native_cx, e4m3_u8_to_f32
# Latency-bound over PCIe, so keep the block small and let many of them be in flight.
_NUM_WARPS = 1
@triton.jit
def _ple_gather_kernel(
table_ptr,
ids_ptr,
out_ptr,
scale,
num_rows,
EMB_DIM: tl.constexpr,
IS_FP8: tl.constexpr,
BLOCK_D: tl.constexpr,
):
row = tl.program_id(0)
idx = tl.load(ids_ptr + row).to(tl.int64)
in_range = (idx >= 0) & (idx < num_rows)
idx = tl.where(in_range, idx, 0)
offsets = tl.arange(0, BLOCK_D)
mask = offsets < EMB_DIM
# the table is a host allocation: rebuild the typed pointer from the raw address
if IS_FP8:
if e4m3_native_cx():
base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.float8e4nv))
values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)
else:
# pre-sm_89 has no fp8e4nv type: load raw bytes and decode in software
base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.uint8))
values = e4m3_u8_to_f32(tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0))
else:
base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.bfloat16))
values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)
values = tl.where(in_range, values * scale, 0.0)
tl.store(
out_ptr + row * EMB_DIM + offsets,
values.to(out_ptr.dtype.element_ty),
mask=mask,
)
def ple_gather_rows(
table_ptr: int,
num_rows: int,
embed_dim: int,
row_ids: torch.Tensor,
out: torch.Tensor,
scale: float = 1.0,
is_fp8: bool = True,
) -> torch.Tensor:
"""Gather ``row_ids`` from the host-resident table at ``table_ptr`` into ``out``.
``row_ids`` is a flat device int tensor; ``out`` is ``[row_ids.numel(), embed_dim]``
bf16 on the same device. ``table_ptr`` is the address the GPU must dereference
(``kernel/pinned.device_ptr``), not necessarily the host ``data_ptr``.
"""
n = row_ids.numel()
assert out.shape == (n, embed_dim) and out.is_contiguous(), out.shape
if n:
_ple_gather_kernel[(n,)](
table_ptr,
row_ids,
out,
float(scale),
num_rows,
EMB_DIM=embed_dim,
IS_FP8=is_fp8,
BLOCK_D=triton.next_power_of_2(embed_dim),
num_warps=_NUM_WARPS,
)
return out
__all__ = ["ple_gather_rows"]