mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(quant)!: compress bf16 weights to df11 at load with --online-quant GLM-4.7 no longer hardwires DF11 attention and embedding; it serves them bf16 unless --online-quant df11 is passed, and FTW files converted with the old DF11 tensors must be reconverted. * fix(quant): skip layers --online-quant cannot convert instead of failing
195 lines
8.5 KiB
Python
195 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Dict
|
|
|
|
import torch
|
|
from freetoken.core import get_global_ctx
|
|
from freetoken.distributed import DistributedCommunicator, get_tp_info
|
|
from freetoken.utils import div_ceil, init_logger, nvtx_annotate
|
|
|
|
from .base import BaseOP
|
|
from .quantization import LayerKind, QuantConfig, quant_method_for
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class VocabParallelEmbedding(BaseOP):
|
|
"""The vocab table, sharded over TP: its ``quant_method`` owns the table, looks rows up, and serves a tied LM head's product."""
|
|
|
|
quant_layer_kind = LayerKind.EMBEDDING
|
|
|
|
def __init__(
|
|
self,
|
|
num_embeddings: int,
|
|
embedding_dim: int,
|
|
embed_scale: float | None = None,
|
|
*,
|
|
quant_config: QuantConfig | None = None,
|
|
prefix: str = "",
|
|
):
|
|
super().__init__()
|
|
tp_info = get_tp_info()
|
|
tp_rank = tp_info.rank
|
|
self.tp_size = tp_info.size
|
|
self.num_embeddings = num_embeddings
|
|
self.num_embeddings_tp = div_ceil(num_embeddings, self.tp_size)
|
|
start_idx = self.num_embeddings_tp * tp_rank
|
|
finish_idx = min(start_idx + self.num_embeddings_tp, num_embeddings)
|
|
self.vocab_range = (start_idx, finish_idx - start_idx)
|
|
self.embedding_dim = embedding_dim
|
|
self.prefix = prefix
|
|
# the LM head subclass is a linear and builds its own method
|
|
self.quant_method = None
|
|
if self.quant_layer_kind is LayerKind.EMBEDDING:
|
|
self.quant_method = quant_method_for(quant_config, self, prefix)
|
|
self.quant_method.create_weights(self)
|
|
# Gemma scales embeddings by sqrt(hidden_size). The scale is materialized in
|
|
# the weight dtype (bf16) to match HF, which downcasts the scalar. The GPU
|
|
# scalar is built lazily (model __init__ runs on the meta device) and cached
|
|
# so it is not reallocated inside a captured CUDA graph.
|
|
self._embed_scale = embed_scale
|
|
self._embed_scale_t: torch.Tensor | None = None
|
|
self._comm = DistributedCommunicator()
|
|
|
|
@nvtx_annotate("Embedding")
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
y = self.quant_method.embed(self, x)
|
|
if self.tp_size > 1:
|
|
y = self._comm.all_reduce(y)
|
|
if self._embed_scale is not None:
|
|
if self._embed_scale_t is None:
|
|
self._embed_scale_t = torch.tensor(
|
|
self._embed_scale, dtype=y.dtype, device=y.device
|
|
)
|
|
y = y * self._embed_scale_t
|
|
return y
|
|
|
|
|
|
class ParallelLMHead(VocabParallelEmbedding):
|
|
"""The head is a linear layer over the vocab shard: its weights come from ``quant_method``
|
|
unless they are tied to the input embedding."""
|
|
|
|
quant_layer_kind = LayerKind.LINEAR
|
|
|
|
def __init__(
|
|
self,
|
|
num_embeddings: int,
|
|
embedding_dim: int,
|
|
bias: bool = False,
|
|
tie_word_embeddings: bool = False,
|
|
tied_embedding: VocabParallelEmbedding | None = None,
|
|
*,
|
|
quant_config: QuantConfig | None = None,
|
|
prefix: str = "",
|
|
):
|
|
super().__init__(num_embeddings, embedding_dim)
|
|
self.has_bias = bias
|
|
self.prefix = prefix
|
|
self.tied_embedding = tied_embedding
|
|
assert (tied_embedding is not None) == tie_word_embeddings
|
|
if tied_embedding is not None and tied_embedding.quant_method.online:
|
|
# the head multiplies by the whole table every step, which an online table would decode in full each time
|
|
logger.info("--online-quant leaves %s as stored: the LM head is tied to it", tied_embedding.prefix)
|
|
tied_embedding.quant_method = quant_method_for(None, tied_embedding, tied_embedding.prefix)
|
|
tied_embedding.quant_method.create_weights(tied_embedding)
|
|
self.in_features = embedding_dim
|
|
self.out_features = self.num_embeddings_tp
|
|
self.output_sizes = (self.num_embeddings_tp,)
|
|
if tied_embedding is None:
|
|
self.quant_method = quant_method_for(quant_config, self, prefix)
|
|
self.quant_method.create_weights(self)
|
|
self.bias = torch.empty(self.num_embeddings_tp) if bias else None
|
|
# set by keep_rows: a head over a subset of the rows, and the token id of each of its rows
|
|
self.kept_rows: tuple[ParallelLMHead, torch.Tensor] | None = None
|
|
|
|
def finalize(self) -> None:
|
|
if self.quant_method is not None:
|
|
self.quant_method.finalize(self)
|
|
|
|
def keep_rows(self, ids: torch.Tensor, quant_config: QuantConfig | None) -> None:
|
|
"""Set ``kept_rows`` to a head with its own weights over only the rows ``ids``, repeated up to a multiple of 128 rows.
|
|
|
|
Call after the weights load and before ``finalize_quant``: a kernel's finalize can rewrite the weights so that rows cannot be picked, and ``finalize_quant`` also finalizes the new head.
|
|
"""
|
|
# scaled_mm needs N % 16, b12x N % 8, and the sm_120 GEMV's fast path N % 128; a repeated row maps to the same token
|
|
ids = ids[torch.arange(div_ceil(ids.numel(), 128) * 128) % ids.numel()]
|
|
rows = ids.numel()
|
|
with torch.device("meta"):
|
|
head = ParallelLMHead(rows, self.embedding_dim, self.has_bias, quant_config=quant_config, prefix=self.prefix)
|
|
full = self.state_dict()
|
|
ids = ids.to(next(iter(full.values())).device)
|
|
for name, want in head.state_dict().items():
|
|
have = full[name]
|
|
if want.dim() == 0:
|
|
kept = have.clone()
|
|
elif want.shape[0] >= rows and have.shape[1:] == want.shape[1:] and have.shape[0] - self.num_embeddings == want.shape[0] - rows:
|
|
kept = have.index_select(0, ids)
|
|
if want.shape[0] > rows:
|
|
# ggml's kernels read past the last row and need zeros there
|
|
kept = torch.cat([kept, kept.new_zeros(want.shape[0] - rows, *kept.shape[1:])])
|
|
else:
|
|
raise NotImplementedError(f"{self.prefix}.{name} is not stored by row (shape {list(have.shape)}), so the head cannot keep a subset of its rows")
|
|
setattr(head, name, kept)
|
|
self.kept_rows = head, ids
|
|
|
|
def load_state_dict(
|
|
self,
|
|
state_dict: Dict[str, torch.Tensor],
|
|
*,
|
|
prefix: str = "",
|
|
_internal: bool = False,
|
|
) -> None:
|
|
if not self.tied_embedding:
|
|
return super().load_state_dict(state_dict, prefix=prefix, _internal=_internal)
|
|
else:
|
|
# pop the lm_head.weights and lm_head.bias if they exist
|
|
possible_weight = f"{prefix}.weight"
|
|
possible_bias = f"{prefix}.bias"
|
|
if possible_weight in state_dict:
|
|
state_dict.pop(possible_weight)
|
|
if possible_bias in state_dict:
|
|
state_dict.pop(possible_bias)
|
|
|
|
def state_dict(
|
|
self,
|
|
*,
|
|
prefix: str = "",
|
|
result: Dict[str, torch.Tensor] | None = None,
|
|
) -> Dict[str, torch.Tensor]:
|
|
if not self.tied_embedding:
|
|
return super().state_dict(prefix=prefix, result=result)
|
|
return {} if result is None else result
|
|
|
|
@nvtx_annotate("LMHead")
|
|
def forward(self, x: torch.Tensor, *, export_hidden: bool = True) -> torch.Tensor:
|
|
"""``export_hidden=False``: the model hands its own final hidden to ``batch.features``."""
|
|
ctx = get_global_ctx()
|
|
batch = ctx.batch
|
|
bs = batch.size
|
|
if export_hidden and (features := batch.features) is not None:
|
|
features.final_hidden(x)
|
|
if batch.is_prefill:
|
|
indices = batch.attn_metadata.get_last_indices(bs)
|
|
x = x[indices].contiguous()
|
|
del indices
|
|
return self.logits(x)
|
|
|
|
def logits(self, x: torch.Tensor) -> torch.Tensor:
|
|
"""``num_embeddings`` logits for each row of ``x``; reads no batch state."""
|
|
if self.tied_embedding is not None:
|
|
logits = self.tied_embedding.quant_method.logits(self.tied_embedding, x, self.bias)
|
|
else:
|
|
logits = self.quant_method.apply(self, x)
|
|
if self.tp_size == 1:
|
|
return logits
|
|
input_shape = logits.shape
|
|
output_tensor = self._comm.all_gather(logits)
|
|
|
|
# keyed on rows, not batch size: a verify batch has several rows per request
|
|
if input_shape[0] == 1:
|
|
return output_tensor.view(1, -1)[:, : self.num_embeddings]
|
|
|
|
output_tensor = output_tensor.view((self.tp_size,) + input_shape)
|
|
output_tensor = output_tensor.permute(1, 0, 2).contiguous()
|
|
output_tensor = output_tensor.reshape(input_shape[:1] + (self.tp_size * input_shape[1],))
|
|
return output_tensor[:, : self.num_embeddings] |