Files
Mukkesh Ganesh ea2916d31c feat(gguf): serve gpt-oss gguf checkpoints on gpt_oss (#838)
* feat(gguf): serve gpt-oss gguf checkpoints on gpt_oss

* fix(gguf): refuse missing gpt-oss bias tensors, honor swa pattern, scale nvfp4 before bias

* fix(gguf): honor gpt-oss yarn truncate, gate the bf16 float cast, scale nvfp4 before the down bias
2026-11-02 08:58:07 -08:00

103 lines
3.4 KiB
Python

from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
import torch
from freetoken.core import get_global_ctx
from freetoken.layers import (
BaseOP,
OPList,
ParallelLMHead,
RMSNormFused,
VocabParallelEmbedding,
)
from freetoken.models.blocks import BaseLLMModel
from freetoken.utils import nvtx_annotate
from .attention import GptOssAttention
from .moe import GptOssMLP
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
class GptOssDecoderLayer(BaseOP):
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.self_attn = GptOssAttention(config, layer_id, prefix=f"{prefix}.self_attn")
self.mlp = GptOssMLP(config, layer_id, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
)
self.post_attention_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
)
self._layer_id = layer_id
@nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
def forward(
self, x: torch.Tensor, residual: torch.Tensor | None = None
) -> Tuple[torch.Tensor, torch.Tensor]:
x, residual = self.input_layernorm.forward(x, residual)
x = self.self_attn.forward(x)
x, residual = self.post_attention_layernorm.forward(x, residual)
x = self.mlp.forward(x)
return x, residual
class GptOssModel(BaseOP):
def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
quant_config=config.quant,
prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
GptOssDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
for layer_id in range(config.num_layers)
]
)
self.norm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
x = self.embed_tokens.forward(input_ids)
residual: torch.Tensor | None = None
for layer in self.layers.op_list:
x, residual = layer.forward(x, residual)
return self.norm.forward(x, residual)[0]
def prepare_for_runtime(self) -> None:
for layer in self.layers.op_list:
layer.mlp.prepare_for_runtime()
class GptOssForCausalLM(BaseLLMModel):
def __init__(self, config: ModelConfig):
self.model = GptOssModel(config)
self.lm_head = ParallelLMHead(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
quant_config=config.quant,
prefix="lm_head",
)
self.config = config
super().__init__()
def forward(self) -> torch.Tensor:
output = self.model.forward(get_global_ctx().batch.input_ids)
return self.lm_head.forward(output)
def prepare_for_runtime(self) -> None:
self.model.prepare_for_runtime()
__all__ = ["GptOssDecoderLayer", "GptOssForCausalLM", "GptOssModel"]