Upload folder using huggingface_hub

.ipynb_checkpoints/config-checkpoint.json

@@ -4,15 +4,19 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModelForSequenceClassification": "modeling_custom.LlamaForSequenceClassificationWithCustomHead",
+    "LlamaForSequenceClassification": "modeling_custom.LlamaForSequenceClassificationWithCustomHead"
+  },
   "bos_token_id": 128000,
   "eos_token_id": [
     128001,
     128008,
     128009
   ],
-  "head_dim": 128,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 4096,
+  "hidden_size": 2048,
   "id2label": {
     "0": "False_Correct:NA",
     "1": "False_Misconception:Adding_across",
@@ -81,7 +85,7 @@
     "64": "True_Neither:NA"
   },
   "initializer_range": 0.02,
-  "intermediate_size": 14336,
+  "intermediate_size": 8192,
   "label2id": {
     "False_Correct:NA": 0,
     "False_Misconception:Adding_across": 1,
@@ -153,35 +157,21 @@
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
-  "num_hidden_layers": 32,
+  "num_hidden_layers": 16,
   "num_key_value_heads": 8,
   "pad_token_id": 128004,
   "pretraining_tp": 1,
-  "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "uint8",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
-  },
+  "problem_type": "single_label_classification",
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
-    "factor": 8.0,
+    "factor": 32.0,
     "high_freq_factor": 4.0,
     "low_freq_factor": 1.0,
     "original_max_position_embeddings": 8192,
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
-  "tie_word_embeddings": false,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.54.1",
   "use_cache": false,

.ipynb_checkpoints/modeling_custom-checkpoint.py

@@ -0,0 +1,110 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Union, Tuple, List
+from transformers import LlamaForSequenceClassification, Cache
+from transformers.modeling_outputs import SequenceClassifierOutputWithPast
+
+
+# 1. Label smoothing cross entropy loss
+class LabelSmoothingCrossEntropy(nn.Module):
+    def __init__(self, eps: float = 0.1):
+        super().__init__()
+        self.eps = eps
+
+    def forward(self, pred, target):
+        num_classes = pred.size(-1)
+        log_preds = F.log_softmax(pred, dim=-1)
+        nll = F.nll_loss(log_preds, target, reduction='none')
+        smooth_loss = -log_preds.sum(dim=-1)
+        loss = (1 - self.eps) * nll + self.eps * smooth_loss / num_classes
+        return loss.mean()
+
+
+# 2. Custom classification head
+class Weights(nn.Module):
+    def __init__(self, hidden_size: int = 4096, num_labels: int = 65):
+        super().__init__()
+        self.fc = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.SELU(),
+            nn.Linear(hidden_size, hidden_size),
+            nn.SELU(),
+            nn.Linear(hidden_size, num_labels)
+        )
+
+    def forward(self, x):
+        return self.fc(x)  # logits, not probabilities
+
+
+# 3. Modified LLaMA model
+class LlamaForSequenceClassificationWithCustomHead(LlamaForSequenceClassification):
+    def __init__(self, config):
+        super().__init__(config)
+        self.weights = Weights(hidden_size=config.hidden_size, num_labels=65)
+        self.loss_fn = LabelSmoothingCrossEntropy(eps=0.1)
+
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = transformer_outputs[0]  # [batch_size, seq_len, hidden_size]
+
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Padding token ID must be set for batch size > 1.")
+
+        # Get sequence length index (last non-padding token)
+        if self.config.pad_token_id is not None and input_ids is not None:
+            sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(dim=-1) - 1
+            sequence_lengths = sequence_lengths % input_ids.shape[-1]
+        else:
+            sequence_lengths = -1  # assume last token if pad token is undefined
+
+        pooled_hidden = hidden_states[torch.arange(batch_size, device=hidden_states.device), sequence_lengths]
+        logits = self.weights(pooled_hidden)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            loss = self.loss_fn(logits, labels)
+
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=logits,  # shape [batch_size, 65]
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )

config.json

@@ -4,15 +4,19 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModelForSequenceClassification": "modeling_custom.LlamaForSequenceClassificationWithCustomHead",
+    "LlamaForSequenceClassification": "modeling_custom.LlamaForSequenceClassificationWithCustomHead"
+  },
   "bos_token_id": 128000,
   "eos_token_id": [
     128001,
     128008,
     128009
   ],
-  "head_dim": 128,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 4096,
+  "hidden_size": 2048,
   "id2label": {
     "0": "False_Correct:NA",
     "1": "False_Misconception:Adding_across",
@@ -81,7 +85,7 @@
     "64": "True_Neither:NA"
   },
   "initializer_range": 0.02,
-  "intermediate_size": 14336,
+  "intermediate_size": 8192,
   "label2id": {
     "False_Correct:NA": 0,
     "False_Misconception:Adding_across": 1,
@@ -153,35 +157,21 @@
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
-  "num_hidden_layers": 32,
+  "num_hidden_layers": 16,
   "num_key_value_heads": 8,
   "pad_token_id": 128004,
   "pretraining_tp": 1,
-  "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "uint8",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
-  },
+  "problem_type": "single_label_classification",
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
-    "factor": 8.0,
+    "factor": 32.0,
     "high_freq_factor": 4.0,
     "low_freq_factor": 1.0,
     "original_max_position_embeddings": 8192,
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
-  "tie_word_embeddings": false,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.54.1",
   "use_cache": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:343e6679cb37fbd9fd7ee26287ee71b692cf91f4d02780526d58e33050fa9c89
-size 4652605573
+oid sha256:9ce145be6003c8e4f68e8306b761a8285fcd8b362b285d19d737ff39df6b0766
+size 2471911936

modeling_custom.py

@@ -0,0 +1,110 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Union, Tuple, List
+from transformers import LlamaForSequenceClassification, Cache
+from transformers.modeling_outputs import SequenceClassifierOutputWithPast
+
+
+# 1. Label smoothing cross entropy loss
+class LabelSmoothingCrossEntropy(nn.Module):
+    def __init__(self, eps: float = 0.1):
+        super().__init__()
+        self.eps = eps
+
+    def forward(self, pred, target):
+        num_classes = pred.size(-1)
+        log_preds = F.log_softmax(pred, dim=-1)
+        nll = F.nll_loss(log_preds, target, reduction='none')
+        smooth_loss = -log_preds.sum(dim=-1)
+        loss = (1 - self.eps) * nll + self.eps * smooth_loss / num_classes
+        return loss.mean()
+
+
+# 2. Custom classification head
+class Weights(nn.Module):
+    def __init__(self, hidden_size: int = 4096, num_labels: int = 65):
+        super().__init__()
+        self.fc = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.SELU(),
+            nn.Linear(hidden_size, hidden_size),
+            nn.SELU(),
+            nn.Linear(hidden_size, num_labels)
+        )
+
+    def forward(self, x):
+        return self.fc(x)  # logits, not probabilities
+
+
+# 3. Modified LLaMA model
+class LlamaForSequenceClassificationWithCustomHead(LlamaForSequenceClassification):
+    def __init__(self, config):
+        super().__init__(config)
+        self.weights = Weights(hidden_size=config.hidden_size, num_labels=65)
+        self.loss_fn = LabelSmoothingCrossEntropy(eps=0.1)
+
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = transformer_outputs[0]  # [batch_size, seq_len, hidden_size]
+
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Padding token ID must be set for batch size > 1.")
+
+        # Get sequence length index (last non-padding token)
+        if self.config.pad_token_id is not None and input_ids is not None:
+            sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(dim=-1) - 1
+            sequence_lengths = sequence_lengths % input_ids.shape[-1]
+        else:
+            sequence_lengths = -1  # assume last token if pad token is undefined
+
+        pooled_hidden = hidden_states[torch.arange(batch_size, device=hidden_states.device), sequence_lengths]
+        logits = self.weights(pooled_hidden)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            loss = self.loss_fn(logits, labels)
+
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=logits,  # shape [batch_size, 65]
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28d7ddb1196b0200360e3a8e353a50a066a80af2e417d8f4ec2352dc8cc92e68
+size 5304