mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
Serve Qwen3.8-Flash-Next (HF model_type qwen4_exp) text-only: 36 GDN + 12 QSA compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE n-gram embedding layer backed by a 47.7 GiB pinned-host table with UVA gather, and 512 NVFP4 / block-fp8 routed experts (top-10) plus a gated shared expert. - attention: qsa_sparse backend (AttnType.QSA) over QSAKVCache -- paged GQA K/V, a 1/ratio compressed index-key slab shadowing the KV pages, and a per-request pending ring sized from index_ratio - kvcache: declarative slot-sibling states (ModelConfig.slot_states) on LinearStatePool carry the PLE conv history and n-gram context through the hybrid-radix snapshot/COW lifecycle - scheduler: hybrid prefill chunks align to the page size so snapshots land on donatable boundaries - kernels: triton kernels adapted from vLLM/SGLang (hc, qsa, ple gather, moe router / shared gate) plus an original radix block top-k; int64 row addressing throughout - moe: non-power-of-2 top-k router, deep-K marlin decode config, one fp8 scale-bank padding rule shared with the AOT row table - engine: the PLE table load reserves its pinned bytes from the pin budget before the expert banks plan their residency
96 lines
3.2 KiB
Python
96 lines
3.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
|
|
# Adapted from SGLang (python/sglang/srt/models/qwen4_exp.py)
|
|
"""UVA row gather for the Qwen3.8-Flash-Next PLE n-gram table.
|
|
|
|
The table (320,001,536 rows x 160, FP8-e4m3 + one scalar scale = 47.7 GiB) stays in pinned
|
|
host memory and the GPU dereferences it in place over PCIe -- at its host VA on Linux/UVA, at
|
|
the mapped device address on WDDM (``kernel/pinned.device_ptr``). One program per requested
|
|
row: read the row, widen to fp32, apply the per-tensor scale, store bf16.
|
|
|
|
Ids outside the table store zeros.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import triton
|
|
import triton.language as tl
|
|
|
|
from freetoken.kernel.triton.e4m3_compat import e4m3_native_cx, e4m3_u8_to_f32
|
|
|
|
# Latency-bound over PCIe, so keep the block small and let many of them be in flight.
|
|
_NUM_WARPS = 1
|
|
|
|
|
|
@triton.jit
|
|
def _ple_gather_kernel(
|
|
table_ptr,
|
|
ids_ptr,
|
|
out_ptr,
|
|
scale,
|
|
num_rows,
|
|
EMB_DIM: tl.constexpr,
|
|
IS_FP8: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
):
|
|
row = tl.program_id(0)
|
|
idx = tl.load(ids_ptr + row).to(tl.int64)
|
|
in_range = (idx >= 0) & (idx < num_rows)
|
|
idx = tl.where(in_range, idx, 0)
|
|
offsets = tl.arange(0, BLOCK_D)
|
|
mask = offsets < EMB_DIM
|
|
# the table is a host allocation: rebuild the typed pointer from the raw address
|
|
if IS_FP8:
|
|
if e4m3_native_cx():
|
|
base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.float8e4nv))
|
|
values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)
|
|
else:
|
|
# pre-sm_89 has no fp8e4nv type: load raw bytes and decode in software
|
|
base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.uint8))
|
|
values = e4m3_u8_to_f32(tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0))
|
|
else:
|
|
base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.bfloat16))
|
|
values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)
|
|
values = tl.where(in_range, values * scale, 0.0)
|
|
tl.store(
|
|
out_ptr + row * EMB_DIM + offsets,
|
|
values.to(out_ptr.dtype.element_ty),
|
|
mask=mask,
|
|
)
|
|
|
|
|
|
def ple_gather_rows(
|
|
table_ptr: int,
|
|
num_rows: int,
|
|
embed_dim: int,
|
|
row_ids: torch.Tensor,
|
|
out: torch.Tensor,
|
|
scale: float = 1.0,
|
|
is_fp8: bool = True,
|
|
) -> torch.Tensor:
|
|
"""Gather ``row_ids`` from the host-resident table at ``table_ptr`` into ``out``.
|
|
|
|
``row_ids`` is a flat device int tensor; ``out`` is ``[row_ids.numel(), embed_dim]``
|
|
bf16 on the same device. ``table_ptr`` is the address the GPU must dereference
|
|
(``kernel/pinned.device_ptr``), not necessarily the host ``data_ptr``.
|
|
"""
|
|
n = row_ids.numel()
|
|
assert out.shape == (n, embed_dim) and out.is_contiguous(), out.shape
|
|
if n:
|
|
_ple_gather_kernel[(n,)](
|
|
table_ptr,
|
|
row_ids,
|
|
out,
|
|
float(scale),
|
|
num_rows,
|
|
EMB_DIM=embed_dim,
|
|
IS_FP8=is_fp8,
|
|
BLOCK_D=triton.next_power_of_2(embed_dim),
|
|
num_warps=_NUM_WARPS,
|
|
)
|
|
return out
|
|
|
|
|
|
__all__ = ["ple_gather_rows"]
|