End of training

README.md

@@ -0,0 +1,57 @@
+---
+library_name: transformers
+license: mit
+base_model: JonusNattapong/gptoss-mini-thaichat
+tags:
+- generated_from_trainer
+model-index:
+- name: gptoss-mini-reasoning
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gptoss-mini-reasoning
+
+This model is a fine-tuned version of [JonusNattapong/gptoss-mini-thaichat](https://huggingface.co/JonusNattapong/gptoss-mini-thaichat) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 200
+- num_epochs: 3
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.57.0.dev0
+- Pytorch 2.8.0+cu126
+- Datasets 4.0.0
+- Tokenizers 0.22.0

configuration_gptoss_mini.py

@@ -0,0 +1,56 @@
+from transformers import PretrainedConfig
+
+
+class GPTMiniConfig(PretrainedConfig):
+    model_type = "gptoss-mini"
+
+    attribute_map = {
+        "num_experts": "num_experts",
+        "top_k": "top_k",
+        "num_hidden_layers": "num_layers"
+    }
+
+    def __init__(
+        self,
+        vocab_size=50000,
+        hidden_size=768,
+        num_layers=6,
+        num_heads=8,
+        num_experts=4,
+        top_k=2,
+        max_position_embeddings=512,
+        intermediate_size=3072,
+        eos_token_id=None,
+        bos_token_id=None,
+        pad_token_id=None,
+        **kwargs
+    ):
+        if top_k > num_experts:
+            raise ValueError(
+                f"top_k ({top_k}) cannot be greater than num_experts ({num_experts})"
+            )
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs
+        )
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.num_experts = num_experts
+        self.top_k = top_k
+        self.max_position_embeddings = max_position_embeddings
+        self.intermediate_size = intermediate_size
+
+        self.num_hidden_layers = num_layers
+
+    def to_dict(self):
+        output = super().to_dict()
+        output["num_experts"] = self.num_experts
+        output["top_k"] = self.top_k
+        output["num_hidden_layers"] = self.num_layers
+        return output

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    3,
+    2
+  ],
+  "max_length": 512,
+  "pad_token_id": 0,
+  "temperature": 0.7,
+  "top_p": 0.9,
+  "transformers_version": "4.57.0.dev0"
+}

modeling_gptoss_mini.py

@@ -0,0 +1,161 @@
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel, GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions
+from .configuration_gptoss_mini import GPTMiniConfig
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, d, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(d))
+        self.eps = eps
+
+    def forward(self, x):
+        norm = x.norm(dim=-1, keepdim=True) * (1.0 / math.sqrt(x.size(-1)))
+        return self.weight * x / (norm + self.eps)
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, d_model, d_ff):
+        super().__init__()
+        self.w1 = nn.Linear(d_model, d_ff)
+        self.w2 = nn.Linear(d_model, d_ff)
+
+    def forward(self, x):
+        return F.silu(self.w1(x)) * self.w2(x)
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, config: GPTMiniConfig):
+        super().__init__()
+        self.qkv = nn.Linear(config.hidden_size, 3 * config.hidden_size)
+        self.o_proj = nn.Linear(config.hidden_size, config.hidden_size)
+        self.num_heads = config.num_heads
+        self.head_dim = config.hidden_size // config.num_heads
+
+    def forward(self, x):
+        B, T, C = x.shape
+        qkv = self.qkv(x).view(B, T, 3, self.num_heads, self.head_dim)
+        q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2]
+
+        attn = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
+        attn = F.softmax(attn, dim=-1)
+        out = attn @ v
+        out = out.reshape(B, T, C)
+        return self.o_proj(out)
+
+
+class MoE(nn.Module):
+    def __init__(self, config: GPTMiniConfig):
+        super().__init__()
+        if config.top_k > config.num_experts:
+            raise ValueError(
+                f"top_k ({config.top_k}) cannot be greater than num_experts ({config.num_experts})"
+            )
+
+        self.experts = nn.ModuleList(
+            [SwiGLU(config.hidden_size, config.intermediate_size) for _ in range(config.num_experts)]
+        )
+        self.gate = nn.Linear(config.hidden_size, config.num_experts)
+        self.top_k = config.top_k
+        self.num_experts = config.num_experts
+
+    def forward(self, x):
+        B, T, C = x.shape
+        scores = F.softmax(self.gate(x), dim=-1)
+
+        current_top_k = min(self.top_k, self.num_experts)
+        topk_scores, topk_idx = torch.topk(scores, current_top_k, dim=-1)
+
+        expert_outputs = torch.stack([expert(x) for expert in self.experts], dim=2)
+        topk_idx_expanded = topk_idx.unsqueeze(-1).expand(-1, -1, -1, C)
+        selected_expert_outputs = torch.gather(expert_outputs, dim=2, index=topk_idx_expanded)
+        topk_scores_expanded = topk_scores.unsqueeze(-1).expand(-1, -1, -1, C)
+        weighted_expert_outputs = selected_expert_outputs * topk_scores_expanded
+        output = torch.sum(weighted_expert_outputs, dim=2)
+
+        return output
+
+
+class Block(nn.Module):
+    def __init__(self, config: GPTMiniConfig):
+        super().__init__()
+        self.ln1 = RMSNorm(config.hidden_size)
+        self.attn = MultiHeadAttention(config)
+        self.ln2 = RMSNorm(config.hidden_size)
+        self.moe = MoE(config)
+
+    def forward(self, x):
+        x = x + self.attn(self.ln1(x))
+        x = x + self.moe(self.ln2(x))
+        return x
+
+
+class GPTMiniForCausalLM(PreTrainedModel, GenerationMixin):
+    config_class = GPTMiniConfig
+
+    def __init__(self, config: GPTMiniConfig):
+        super().__init__(config)
+        self.embed = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.pos_embed = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
+        self.ln_f = RMSNorm(config.hidden_size)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed
+
+    def set_input_embeddings(self, new_embeddings):
+        self.embed = new_embeddings
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def tie_weights(self):
+        self._tie_or_clone_weights(self.lm_head, self.embed)
+
+    def forward(
+        self,
+        input_ids,
+        labels=None,
+        attention_mask=None,
+        token_type_ids=None,
+        past_key_values=None,
+        use_cache: bool = False,
+        cache_position=None,     
+        **kwargs                
+    ):
+        B, T = input_ids.shape
+        pos = torch.arange(0, T, device=input_ids.device).unsqueeze(0)
+        x = self.embed(input_ids) + self.pos_embed(pos)
+
+        for block in self.blocks:
+            x = block(x)
+
+        x = self.ln_f(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits.view(-1, logits.size(-1)),
+                labels.view(-1),
+                ignore_index=-100
+            )
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=logits,
+            past_key_values=past_key_values if use_cache else None,
+            hidden_states=None,
+            attentions=None,
+        )