mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* refactor(quant): config, scheme and method layers for checkpoint quantization QuantConfig parses the checkpoint's quantization_config into a QuantScheme per projection; every Linear*, ParallelLMHead and MoE layer takes quant_config and a prefix, declares its tensors through a QuantMethod and selects its kernel at construction. MoE kernels own the expert bank layout, pack and apply, so the offload cache, the FTW converter and the CPU executor follow one layout. - --quant-backend layer[.kind]=name selects kernels (--nvfp4-backend forwards to it) - --moe-strategy replaces --moe-backend; the old spelling stays as an alias - --moe-cpu-layers auto for Windows/WSL pin budgets; --moe-cache-auto respects the kernel's slot limit - removes the per-family quantized layer classes, bank providers, format env switches and GLM runtime fp8
177 lines
7.8 KiB
Python
177 lines
7.8 KiB
Python
"""Per-tensor FP8 linear: the W8A16 kernels against the dequant reference, and the W8A8
|
|
(torch._scaled_mm) path against a W8A8 reference.
|
|
|
|
Which of the two runs is fixed by the deployment -- an ``input_scale`` in the checkpoint plus
|
|
sm_89+ -- and never by M, so each is checked against the reference matching its own contract:
|
|
W8A16 keeps the activation in bf16 and matches the exact dequant reference, W8A8 quantizes it
|
|
to fp8 and is held to a reference that applies the same quantization.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
if not torch.cuda.is_available(): # pragma: no cover
|
|
pytest.skip("CUDA required", allow_module_level=True)
|
|
|
|
from freetoken.kernel.triton.e4m3_compat import e4m3_native
|
|
|
|
DEV = "cuda"
|
|
FP8 = torch.float8_e4m3fn
|
|
|
|
|
|
def _quant_parts(part_rows: list[int], K: int, seed: int = 0):
|
|
"""A fused per-tensor-FP8 weight: each part quantized under its own scalar, exactly how
|
|
modelopt stores q/k/v (and how the loader concatenates them into a per-row vector)."""
|
|
torch.manual_seed(seed)
|
|
N = sum(part_rows)
|
|
wf = torch.randn(N, K, device=DEV) * 0.05
|
|
rows, qs, scales = 0, [], []
|
|
for p in part_rows:
|
|
block = wf[rows : rows + p]
|
|
s = block.abs().max() / 448.0
|
|
qs.append((block / s).clamp(-448, 448).to(FP8))
|
|
scales.append(s.expand(p))
|
|
rows += p
|
|
return torch.cat(qs, 0), torch.cat(scales).contiguous().float()
|
|
|
|
|
|
def _dequant(w8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
|
return w8.to(torch.float32) * scale[:, None]
|
|
|
|
|
|
# 1 = split-K GEMV; 2..16 = decode batch (the CUDA-graph ladder); 64/300 = prefill.
|
|
@pytest.mark.parametrize("M", [1, 2, 4, 8, 16, 64, 300])
|
|
# (5120, 14336) and (5120, 16384) are Qwen3.5-27B's fused qkv_proj / in_proj_qkvz;
|
|
# (1024, 6144) is a standalone o_proj shape; 6112 leaves a k-mask tail.
|
|
@pytest.mark.parametrize("K,part_rows", [
|
|
(5120, [12288, 1024, 1024]),
|
|
(1024, [6144]),
|
|
(6112, [512, 128]),
|
|
])
|
|
def test_w8a16_matches_dequant_reference(M: int, K: int, part_rows: list[int]):
|
|
"""Without an ``input_scale`` every M stays on the W8A16 kernels, activation exact."""
|
|
from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
|
|
|
|
w8, scale = _quant_parts(part_rows, K, seed=M)
|
|
x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)
|
|
y = fp8_pertensor_linear(x, w8, scale)
|
|
y_ref = (x.float() @ _dequant(w8, scale).t()).to(torch.bfloat16)
|
|
rel = (y.float() - y_ref.float()).abs().max() / y_ref.float().abs().max().clamp(min=1e-6)
|
|
assert rel.item() < 2e-2, rel.item()
|
|
|
|
|
|
@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
|
|
@pytest.mark.parametrize("M", [1, 2, 4, 16, 64])
|
|
@pytest.mark.parametrize("part_rows,uniform", [
|
|
([12288, 1024, 1024], False), # fused -> piecewise-constant scale -> row-wise
|
|
([6144], True), # standalone -> one scalar -> tensor-wise
|
|
])
|
|
def test_w8a8_matches_w8a8_reference(M: int, part_rows: list[int], uniform: bool):
|
|
from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
|
|
|
|
K = 2048
|
|
w8, scale = _quant_parts(part_rows, K, seed=M)
|
|
x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)
|
|
input_scale = (x.abs().max().float() / 448.0).reshape(())
|
|
|
|
y = fp8_pertensor_linear(x, w8, scale, None, input_scale, uniform)
|
|
|
|
xq = (x.float() / input_scale).clamp(-448, 448).to(FP8)
|
|
y_ref = (xq.to(torch.float32) * input_scale) @ _dequant(w8, scale).t()
|
|
rel = ((y.float() - y_ref).norm() / y_ref.norm()).item()
|
|
assert rel < 1e-2, rel
|
|
|
|
|
|
@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
|
|
def test_batch_size_does_not_change_the_numeric_scheme():
|
|
"""A deployment that can run W8A8 must run it at every M, so that a reply reproduces at
|
|
bs=1 regardless of how many other requests shared its forward. Feeding the same row alone
|
|
and as part of a batch must therefore agree bit-for-bit."""
|
|
from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
|
|
|
|
K, part_rows = 2048, [1024, 256]
|
|
w8, scale = _quant_parts(part_rows, K)
|
|
x = torch.randn(8, K, device=DEV, dtype=torch.bfloat16)
|
|
input_scale = (x.abs().max().float() / 448.0).reshape(())
|
|
|
|
batched = fp8_pertensor_linear(x, w8, scale, None, input_scale, False)
|
|
alone = fp8_pertensor_linear(x[:1], w8, scale, None, input_scale, False)
|
|
assert torch.equal(alone, batched[:1])
|
|
|
|
|
|
@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
|
|
@pytest.mark.parametrize("M", [1, 4, 64, 300])
|
|
def test_per_part_path_matches_rowwise(M: int, monkeypatch):
|
|
"""Where row-wise ``_scaled_mm`` is unsafe a fused projection runs one tensor-wise GEMM per
|
|
part instead. Same scheme, so the two paths agree up to accumulation order (~7e-4)."""
|
|
import freetoken.kernel.triton.fp8_pertensor_linear as mod
|
|
|
|
K, part_rows = 2048, [1024, 256, 256]
|
|
w8, scale = _quant_parts(part_rows, K, seed=M)
|
|
x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)
|
|
input_scale = (x.abs().max().float() / 448.0).reshape(())
|
|
|
|
monkeypatch.setattr(mod, "rowwise_scaled_mm_ok", lambda: True)
|
|
y_row = mod.fp8_pertensor_linear(x, w8, scale, None, input_scale, False)
|
|
monkeypatch.setattr(mod, "rowwise_scaled_mm_ok", lambda: False)
|
|
y_part = mod.fp8_pertensor_linear(x, w8, scale, None, input_scale, False)
|
|
rel = ((y_part.float() - y_row.float()).norm() / y_row.float().norm()).item()
|
|
assert rel < 2e-3, rel
|
|
|
|
|
|
@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
|
|
def test_fused_layer_forward_on_a_side_stream_completes():
|
|
"""Regression for #182 / #72 / #220: on sm_89 with torch < 2.12 a fused FP8 projection's
|
|
row-wise ``_scaled_mm`` issued from a non-default stream stalls the GPU (PyTorch's
|
|
CUTLASS row-wise kernel ignored the current stream; fixed upstream in 2.12). The layer
|
|
must take a path that completes on every supported build. Runs in a subprocess so a
|
|
stall fails the test instead of hanging the session."""
|
|
script = textwrap.dedent("""
|
|
import os, time, torch
|
|
from freetoken.kernel.triton.fp8_pertensor_linear import (
|
|
FP8, fp8_pertensor_linear, rowwise_scaled_mm_ok, weight_scale_segments,
|
|
)
|
|
|
|
torch.manual_seed(0)
|
|
K, parts = 2048, [8192, 512, 512] # a prefill-sized fused qkv
|
|
w8 = (torch.randn(sum(parts), K, device="cuda") * 8).clamp(-448, 448).to(FP8)
|
|
scale = torch.cat([torch.full((p,), 0.01 * (i + 1), device="cuda")
|
|
for i, p in enumerate(parts)])
|
|
input_scale = torch.tensor(0.02, device="cuda")
|
|
# same load-time decisions the torch fp8_tensor kernel makes in finalize
|
|
segments = weight_scale_segments(scale)
|
|
rowwise_scaled_mm_ok()
|
|
x = torch.randn(2010, K, device="cuda", dtype=torch.bfloat16) # #182 shape
|
|
torch.cuda.synchronize()
|
|
|
|
stream = torch.cuda.Stream()
|
|
events = []
|
|
with torch.cuda.stream(stream):
|
|
for _ in range(128):
|
|
fp8_pertensor_linear(x, w8, scale, None, input_scale, False, scale_segments=segments)
|
|
ev = torch.cuda.Event()
|
|
ev.record(stream)
|
|
events.append(ev)
|
|
deadline = time.monotonic() + 30
|
|
done = 0
|
|
while time.monotonic() < deadline:
|
|
while done < len(events) and events[done].query():
|
|
done += 1
|
|
if done == len(events):
|
|
print("completed", done, flush=True)
|
|
os._exit(0)
|
|
time.sleep(0.05)
|
|
print("stalled at", done, "of", len(events), flush=True)
|
|
os._exit(124) # a normal exit would wait on the stuck kernel
|
|
""")
|
|
proc = subprocess.run(
|
|
[sys.executable, "-c", script], capture_output=True, text=True, timeout=300,
|
|
)
|
|
assert proc.returncode == 0, f"rc={proc.returncode}\n{proc.stdout}\n{proc.stderr[-2000:]}"
|