MyModel-sft / model.py
HappyBulingling's picture
Upload MyModel checkpoint
72f5707 verified
Raw History Blame Contribute Delete
11 kB
"""MyModel backbone and causal language-model head."""
from __future__ import annotations
import math
from collections.abc import Sequence
from typing import Iterable
import torch
import torch.nn.functional as F
from torch import nn
from torch.utils.checkpoint import checkpoint
from transformers import GenerationMixin, PreTrainedModel
from transformers.modeling_outputs import CausalLMOutputWithPast
from .block import Block
from .config import MyModelConfig
from .norm import RMSNorm
from .rope import precompute_rope
def chunked_cross_entropy(
hidden_states: torch.Tensor,
labels: torch.Tensor,
lm_head: nn.Linear,
chunk_size: int,
ignore_index: int = -100,
) -> torch.Tensor:
"""Next-token cross entropy without materializing all logits at once."""
if hidden_states.shape[:2] != labels.shape:
raise ValueError(
f"hidden_states {tuple(hidden_states.shape[:2])} and labels "
f"{tuple(labels.shape)} must agree on batch and sequence length"
)
if chunk_size <= 0:
raise ValueError("chunk_size must be positive")
hidden = hidden_states[:, :-1, :]
target = labels[:, 1:]
total_loss = hidden_states.new_zeros(())
total_tokens = 0
for start in range(0, hidden.shape[1], chunk_size):
chunk_hidden = hidden[:, start : start + chunk_size, :]
chunk_target = target[:, start : start + chunk_size]
logits = lm_head(chunk_hidden)
total_loss = total_loss + F.cross_entropy(
logits.reshape(-1, logits.shape[-1]),
chunk_target.reshape(-1),
ignore_index=ignore_index,
reduction="sum",
)
total_tokens += int((chunk_target != ignore_index).sum())
return total_loss / max(total_tokens, 1)
class MyModelModel(nn.Module):
def __init__(self, config: MyModelConfig):
super().__init__()
self.config = config
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
self.dropout = nn.Dropout(config.dropout)
self.layers = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)])
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
cos, sin = precompute_rope(
config.head_dim,
config.max_position_embeddings,
config.rope_theta,
)
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
self.gradient_checkpointing = False
def gradient_checkpointing_enable(self) -> None:
self.gradient_checkpointing = True
def gradient_checkpointing_disable(self) -> None:
self.gradient_checkpointing = False
def forward(
self,
input_ids: torch.Tensor,
attention_mask: torch.Tensor | None = None,
past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None,
use_cache: bool = False,
) -> tuple[torch.Tensor, tuple[tuple[torch.Tensor, torch.Tensor], ...] | None]:
batch, seq_len = input_ids.shape
if past_key_values is None:
past_key_values = [None] * len(self.layers)
else:
past_key_values = list(past_key_values)
if len(past_key_values) != len(self.layers):
raise ValueError(
f"expected {len(self.layers)} past key/value entries, "
f"got {len(past_key_values)}"
)
start_pos = 0 if past_key_values[0] is None else past_key_values[0][0].shape[1]
if start_pos + seq_len > self.config.max_position_embeddings:
raise ValueError(
f"sequence length {start_pos + seq_len} exceeds "
f"max_position_embeddings {self.config.max_position_embeddings}"
)
hidden_states = self.dropout(self.embed_tokens(input_ids))
cos = self.rope_cos[start_pos : start_pos + seq_len]
sin = self.rope_sin[start_pos : start_pos + seq_len]
presents: list[tuple[torch.Tensor, torch.Tensor] | None] = []
for layer, past_key_value in zip(self.layers, past_key_values):
if self.gradient_checkpointing and self.training:
hidden_states, _ = checkpoint(
layer,
hidden_states,
cos,
sin,
past_key_value,
False,
attention_mask,
use_reentrant=False,
)
presents.append(None)
else:
hidden_states, present = layer(
hidden_states,
cos,
sin,
past_key_value,
use_cache,
attention_mask,
)
presents.append(present)
hidden_states = self.norm(hidden_states)
if not use_cache:
return hidden_states, None
return hidden_states, tuple(presents)
class MyModelForCausalLM(PreTrainedModel, GenerationMixin):
config_class = MyModelConfig
_tied_weights_keys = ["lm_head.weight"]
def __init__(self, config: MyModelConfig):
super().__init__(config)
self.model = MyModelModel(config)
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
if config.tie_word_embeddings:
self.lm_head.weight = self.model.embed_tokens.weight
self.post_init()
self._scale_residual_projections()
def _init_weights(self, module: nn.Module) -> None:
if isinstance(module, nn.Linear):
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
if module.bias is not None:
module.bias.data.zero_()
elif isinstance(module, nn.Embedding):
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
def _scale_residual_projections(self) -> None:
scale = 1.0 / math.sqrt(2 * self.config.num_hidden_layers)
with torch.no_grad():
for layer in self.model.layers:
layer.self_attn.o_proj.weight.mul_(scale)
layer.mlp.down_proj.weight.mul_(scale)
def gradient_checkpointing_enable(self) -> None:
self.model.gradient_checkpointing_enable()
def gradient_checkpointing_disable(self) -> None:
self.model.gradient_checkpointing_disable()
def forward(
self,
input_ids: torch.Tensor,
attention_mask: torch.Tensor | None = None,
past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None,
use_cache: bool = False,
labels: torch.Tensor | None = None,
chunk_size: int = 0,
logits_to_keep: int = 0,
**kwargs,
) -> CausalLMOutputWithPast:
hidden_states, presents = self.model(
input_ids,
attention_mask=attention_mask,
past_key_values=past_key_values,
use_cache=use_cache,
)
loss = None
logits = None
if labels is not None and chunk_size and chunk_size > 0:
loss = chunked_cross_entropy(hidden_states, labels, self.lm_head, chunk_size)
else:
logits = self.lm_head(hidden_states)
if labels is not None:
shift_logits = logits[:, :-1, :]
shift_labels = labels[:, 1:]
loss = F.cross_entropy(
shift_logits.reshape(-1, shift_logits.shape[-1]),
shift_labels.reshape(-1),
ignore_index=-100,
)
if logits_to_keep > 0:
logits = logits[:, -logits_to_keep:, :]
return CausalLMOutputWithPast(
loss=loss,
logits=logits,
past_key_values=presents,
hidden_states=hidden_states,
attentions=None,
)
@torch.inference_mode()
def generate(
self,
input_ids: torch.Tensor,
max_new_tokens: int = 64,
temperature: float = 1.0,
top_k: int = 0,
top_p: float = 1.0,
eos_token_id: int | Sequence[int] | None = None,
do_sample: bool = True,
use_cache: bool = True,
) -> torch.Tensor:
if temperature <= 0:
raise ValueError("temperature must be positive")
if eos_token_id is None:
stop_ids = [int(self.config.eos_token_id)]
im_end = getattr(self.config, "im_end_token_id", None)
if im_end is not None:
stop_ids.append(int(im_end))
elif isinstance(eos_token_id, int):
stop_ids = [eos_token_id]
else:
stop_ids = [int(token_id) for token_id in eos_token_id]
stop_ids = list(dict.fromkeys(stop_ids))
if not stop_ids:
raise ValueError("eos_token_id must contain at least one token id")
stop_tensor = torch.tensor(
stop_ids, dtype=torch.long, device=input_ids.device
)
fill_id = stop_ids[0]
past_key_values = None
finished = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device)
for _ in range(max_new_tokens):
past_len = 0 if past_key_values is None else past_key_values[0][0].shape[1]
outputs = self(
input_ids[:, past_len:],
past_key_values=past_key_values,
use_cache=use_cache,
)
logits = outputs.logits[:, -1, :] / temperature
if top_k > 0:
threshold = torch.topk(logits, min(top_k, logits.shape[-1]), dim=-1).values[..., -1:]
logits = logits.masked_fill(logits < threshold, float("-inf"))
if top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
cumulative = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
remove = cumulative > top_p
remove[..., 1:] = remove[..., :-1].clone()
remove[..., 0] = False
logits = logits.masked_fill(remove.scatter(1, sorted_indices, remove), float("-inf"))
if do_sample:
probs = torch.softmax(logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1)
else:
next_token = torch.argmax(logits, dim=-1, keepdim=True)
next_token = torch.where(
finished.unsqueeze(-1),
next_token.new_full((next_token.shape[0], 1), fill_id),
next_token,
)
input_ids = torch.cat((input_ids, next_token), dim=-1)
past_key_values = outputs.past_key_values if use_cache else None
finished |= (next_token == stop_tensor).any(dim=-1).reshape(-1)
if finished.all():
break
return input_ids