"""MyModel backbone and causal language-model head.""" from __future__ import annotations import math from collections.abc import Sequence from typing import Iterable import torch import torch.nn.functional as F from torch import nn from torch.utils.checkpoint import checkpoint from transformers import GenerationMixin, PreTrainedModel from transformers.modeling_outputs import CausalLMOutputWithPast from .block import Block from .config import MyModelConfig from .norm import RMSNorm from .rope import precompute_rope def chunked_cross_entropy( hidden_states: torch.Tensor, labels: torch.Tensor, lm_head: nn.Linear, chunk_size: int, ignore_index: int = -100, ) -> torch.Tensor: """Next-token cross entropy without materializing all logits at once.""" if hidden_states.shape[:2] != labels.shape: raise ValueError( f"hidden_states {tuple(hidden_states.shape[:2])} and labels " f"{tuple(labels.shape)} must agree on batch and sequence length" ) if chunk_size <= 0: raise ValueError("chunk_size must be positive") hidden = hidden_states[:, :-1, :] target = labels[:, 1:] total_loss = hidden_states.new_zeros(()) total_tokens = 0 for start in range(0, hidden.shape[1], chunk_size): chunk_hidden = hidden[:, start : start + chunk_size, :] chunk_target = target[:, start : start + chunk_size] logits = lm_head(chunk_hidden) total_loss = total_loss + F.cross_entropy( logits.reshape(-1, logits.shape[-1]), chunk_target.reshape(-1), ignore_index=ignore_index, reduction="sum", ) total_tokens += int((chunk_target != ignore_index).sum()) return total_loss / max(total_tokens, 1) class MyModelModel(nn.Module): def __init__(self, config: MyModelConfig): super().__init__() self.config = config self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) self.dropout = nn.Dropout(config.dropout) self.layers = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)]) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) cos, sin = precompute_rope( config.head_dim, config.max_position_embeddings, config.rope_theta, ) self.register_buffer("rope_cos", cos, persistent=False) self.register_buffer("rope_sin", sin, persistent=False) self.gradient_checkpointing = False def gradient_checkpointing_enable(self) -> None: self.gradient_checkpointing = True def gradient_checkpointing_disable(self) -> None: self.gradient_checkpointing = False def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None, use_cache: bool = False, ) -> tuple[torch.Tensor, tuple[tuple[torch.Tensor, torch.Tensor], ...] | None]: batch, seq_len = input_ids.shape if past_key_values is None: past_key_values = [None] * len(self.layers) else: past_key_values = list(past_key_values) if len(past_key_values) != len(self.layers): raise ValueError( f"expected {len(self.layers)} past key/value entries, " f"got {len(past_key_values)}" ) start_pos = 0 if past_key_values[0] is None else past_key_values[0][0].shape[1] if start_pos + seq_len > self.config.max_position_embeddings: raise ValueError( f"sequence length {start_pos + seq_len} exceeds " f"max_position_embeddings {self.config.max_position_embeddings}" ) hidden_states = self.dropout(self.embed_tokens(input_ids)) cos = self.rope_cos[start_pos : start_pos + seq_len] sin = self.rope_sin[start_pos : start_pos + seq_len] presents: list[tuple[torch.Tensor, torch.Tensor] | None] = [] for layer, past_key_value in zip(self.layers, past_key_values): if self.gradient_checkpointing and self.training: hidden_states, _ = checkpoint( layer, hidden_states, cos, sin, past_key_value, False, attention_mask, use_reentrant=False, ) presents.append(None) else: hidden_states, present = layer( hidden_states, cos, sin, past_key_value, use_cache, attention_mask, ) presents.append(present) hidden_states = self.norm(hidden_states) if not use_cache: return hidden_states, None return hidden_states, tuple(presents) class MyModelForCausalLM(PreTrainedModel, GenerationMixin): config_class = MyModelConfig _tied_weights_keys = ["lm_head.weight"] def __init__(self, config: MyModelConfig): super().__init__(config) self.model = MyModelModel(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) if config.tie_word_embeddings: self.lm_head.weight = self.model.embed_tokens.weight self.post_init() self._scale_residual_projections() def _init_weights(self, module: nn.Module) -> None: if isinstance(module, nn.Linear): module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) if module.bias is not None: module.bias.data.zero_() elif isinstance(module, nn.Embedding): module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) def _scale_residual_projections(self) -> None: scale = 1.0 / math.sqrt(2 * self.config.num_hidden_layers) with torch.no_grad(): for layer in self.model.layers: layer.self_attn.o_proj.weight.mul_(scale) layer.mlp.down_proj.weight.mul_(scale) def gradient_checkpointing_enable(self) -> None: self.model.gradient_checkpointing_enable() def gradient_checkpointing_disable(self) -> None: self.model.gradient_checkpointing_disable() def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None, use_cache: bool = False, labels: torch.Tensor | None = None, chunk_size: int = 0, logits_to_keep: int = 0, **kwargs, ) -> CausalLMOutputWithPast: hidden_states, presents = self.model( input_ids, attention_mask=attention_mask, past_key_values=past_key_values, use_cache=use_cache, ) loss = None logits = None if labels is not None and chunk_size and chunk_size > 0: loss = chunked_cross_entropy(hidden_states, labels, self.lm_head, chunk_size) else: logits = self.lm_head(hidden_states) if labels is not None: shift_logits = logits[:, :-1, :] shift_labels = labels[:, 1:] loss = F.cross_entropy( shift_logits.reshape(-1, shift_logits.shape[-1]), shift_labels.reshape(-1), ignore_index=-100, ) if logits_to_keep > 0: logits = logits[:, -logits_to_keep:, :] return CausalLMOutputWithPast( loss=loss, logits=logits, past_key_values=presents, hidden_states=hidden_states, attentions=None, ) @torch.inference_mode() def generate( self, input_ids: torch.Tensor, max_new_tokens: int = 64, temperature: float = 1.0, top_k: int = 0, top_p: float = 1.0, eos_token_id: int | Sequence[int] | None = None, do_sample: bool = True, use_cache: bool = True, ) -> torch.Tensor: if temperature <= 0: raise ValueError("temperature must be positive") if eos_token_id is None: stop_ids = [int(self.config.eos_token_id)] im_end = getattr(self.config, "im_end_token_id", None) if im_end is not None: stop_ids.append(int(im_end)) elif isinstance(eos_token_id, int): stop_ids = [eos_token_id] else: stop_ids = [int(token_id) for token_id in eos_token_id] stop_ids = list(dict.fromkeys(stop_ids)) if not stop_ids: raise ValueError("eos_token_id must contain at least one token id") stop_tensor = torch.tensor( stop_ids, dtype=torch.long, device=input_ids.device ) fill_id = stop_ids[0] past_key_values = None finished = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device) for _ in range(max_new_tokens): past_len = 0 if past_key_values is None else past_key_values[0][0].shape[1] outputs = self( input_ids[:, past_len:], past_key_values=past_key_values, use_cache=use_cache, ) logits = outputs.logits[:, -1, :] / temperature if top_k > 0: threshold = torch.topk(logits, min(top_k, logits.shape[-1]), dim=-1).values[..., -1:] logits = logits.masked_fill(logits < threshold, float("-inf")) if top_p < 1.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1) cumulative = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1) remove = cumulative > top_p remove[..., 1:] = remove[..., :-1].clone() remove[..., 0] = False logits = logits.masked_fill(remove.scatter(1, sorted_indices, remove), float("-inf")) if do_sample: probs = torch.softmax(logits, dim=-1) next_token = torch.multinomial(probs, num_samples=1) else: next_token = torch.argmax(logits, dim=-1, keepdim=True) next_token = torch.where( finished.unsqueeze(-1), next_token.new_full((next_token.shape[0], 1), fill_id), next_token, ) input_ids = torch.cat((input_ids, next_token), dim=-1) past_key_values = outputs.past_key_values if use_cache else None finished |= (next_token == stop_tensor).any(dim=-1).reshape(-1) if finished.all(): break return input_ids