mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
93 lines
3.1 KiB
Python
93 lines
3.1 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Tuple
|
|
|
|
import torch
|
|
from freetoken.core import get_global_ctx
|
|
from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNormFused, VocabParallelEmbedding
|
|
from freetoken.utils import nvtx_annotate
|
|
|
|
from freetoken.models.blocks import BaseLLMModel, GatedMLP as LlamaMLP
|
|
|
|
from .attention import LlamaAttention as LlamaAttn
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.models.config import ModelConfig
|
|
|
|
|
|
class LlamaDecoderLayer(BaseOP):
|
|
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
|
|
self.self_attn = LlamaAttn(config, layer_id, prefix=f"{prefix}.self_attn")
|
|
self.mlp = LlamaMLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")
|
|
self.input_layernorm = RMSNormFused(
|
|
size=config.hidden_size,
|
|
eps=config.rms_norm_eps,
|
|
)
|
|
self.post_attention_layernorm = RMSNormFused(
|
|
size=config.hidden_size,
|
|
eps=config.rms_norm_eps,
|
|
)
|
|
|
|
self._layer_id = layer_id
|
|
|
|
@nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
|
|
def forward(
|
|
self,
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor | None = None,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
x, residual = self.input_layernorm.forward(x, residual)
|
|
x = self.self_attn.forward(x)
|
|
x, residual = self.post_attention_layernorm.forward(x, residual)
|
|
x = self.mlp.forward(x)
|
|
return x, residual
|
|
|
|
|
|
class LlamaModel(BaseOP):
|
|
def __init__(self, config: ModelConfig, *, prefix: str = "model"):
|
|
self.embed_tokens = VocabParallelEmbedding(
|
|
num_embeddings=config.vocab_size,
|
|
embedding_dim=config.hidden_size,
|
|
quant_config=config.quant,
|
|
prefix=f"{prefix}.embed_tokens",
|
|
)
|
|
self.layers = OPList(
|
|
[
|
|
LlamaDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
|
|
for layer_id in range(config.num_layers)
|
|
]
|
|
)
|
|
self.norm = RMSNormFused(
|
|
size=config.hidden_size,
|
|
eps=config.rms_norm_eps,
|
|
)
|
|
|
|
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
|
x = self.embed_tokens.forward(input_ids)
|
|
residual: torch.Tensor | None = None
|
|
for layer in self.layers.op_list:
|
|
x, residual = layer.forward(x, residual)
|
|
return self.norm.forward(x, residual)[0]
|
|
|
|
|
|
class LlamaForCausalLM(BaseLLMModel):
|
|
def __init__(self, config: ModelConfig):
|
|
self.model = LlamaModel(config)
|
|
self.lm_head = ParallelLMHead(
|
|
num_embeddings=config.vocab_size,
|
|
embedding_dim=config.hidden_size,
|
|
tie_word_embeddings=config.tie_word_embeddings,
|
|
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
|
|
quant_config=config.quant,
|
|
prefix="lm_head",
|
|
)
|
|
super().__init__()
|
|
|
|
def forward(self) -> torch.Tensor:
|
|
output = self.model.forward(get_global_ctx().batch.input_ids)
|
|
logits = self.lm_head.forward(output)
|
|
return logits
|
|
|
|
|
|
__all__ = ["LlamaForCausalLM"]
|