Upload 47 files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Deep_Learning_Project/A24-Y2-DEEP-LEARNING-Project.pdf filter=lfs diff=lfs merge=lfs -text
+Deep_Learning_Project/Deep[[:space:]]Learning[[:space:]]Project[[:space:]]Report.docx filter=lfs diff=lfs merge=lfs -text
+Deep_Learning_Project/Deep[[:space:]]Learning[[:space:]]Project[[:space:]]Report.pdf filter=lfs diff=lfs merge=lfs -text
+Deep_Learning_Project/eda_plots.png filter=lfs diff=lfs merge=lfs -text

Deep_Learning_Project/A24-Y2-DEEP-LEARNING-Project.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d248954892441dbf8de6cb3c8315718e020879401296dd7d1597cd82fe40dce2
+size 230345

Deep_Learning_Project/Deep Learning Project Report.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccbe55fe11859c664c37d29a179ce14404ad4084a63ad430daff5aff2ae56da0
+size 236057

Deep_Learning_Project/Deep Learning Project Report.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1381d9fa4aa0351d88ff4c151941d46d177e87ba033d0947bffce069fdb251f3
+size 357842

Deep_Learning_Project/README.md

@@ -0,0 +1,31 @@
+# Deep Learning Project: Spam Detection with DistilBERT
+
+This repository contains the code and resources for the Deep Learning project on Spam Detection.
+
+## Project Structure
+- `mail_data.csv`: The dataset used for training and evaluation.
+- `eda_script.py`: Script for Exploratory Data Analysis and visualization.
+- `train_model_hf.py`: Main training script using Hugging Face Trainer and DistilBERT.
+- `evaluate_final.py`: Script for final evaluation from the best model checkpoint.
+- `eda_plots.png`: Visualizations generated during EDA.
+- `results.txt`: Detailed evaluation metrics and confusion matrix.
+- `Deep_Learning_Project_Report.pdf`: The final project report (15-17 pages equivalent).
+
+## Requirements
+- Python 3.11+
+- PyTorch
+- Transformers
+- Datasets
+- Scikit-learn
+- Pandas
+- Matplotlib
+- Seaborn
+- Accelerate
+
+## How to Run
+1.  **EDA**: Run `python3 eda_script.py` to see the data distribution.
+2.  **Training**: Run `python3 train_model_hf.py` to fine-tune the DistilBERT model.
+3.  **Evaluation**: Run `python3 evaluate_final.py` to get the final performance metrics.
+
+## Results
+The model achieves **99.10% accuracy** on the test set with an **F1-score of 96.58%** for the spam class.

Deep_Learning_Project/app.py

@@ -0,0 +1,74 @@
+import gradio as gr
+import torch
+from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
+import torch.nn.functional as F
+
+# 1. Load the model and tokenizer from your saved directory
+# If uploading to Hugging Face Spaces, ensure your saved_model folder is uploaded too!
+model_path = "./saved_model" # Update this path to where your model is saved
+
+try:
+    tokenizer = DistilBertTokenizer.from_pretrained(model_path)
+    model = DistilBertForSequenceClassification.from_pretrained(model_path)
+    model.eval() # Set model to evaluation mode
+except Exception as e:
+    # Fallback to base model if you are just testing the code without your fine-tuned weights
+    # (Remove this try-except block in your final version, it's just to prevent crashes if path is wrong)
+    print("Could not load fine-tuned model, loading base model for demonstration purposes...")
+    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=2)
+    model.eval()
+
+# 2. Define the prediction function
+def predict_spam(message):
+    if not message.strip():
+        return {"Please enter a message": 1.0}
+
+    # Tokenize input
+    inputs = tokenizer(message, return_tensors="pt", truncation=True, padding=True, max_length=128)
+    
+    # Get model prediction
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        
+    # Apply softmax to get probabilities
+    probabilities = F.softmax(logits, dim=1).squeeze()
+    
+    # Our labels are: 0 -> Ham, 1 -> Spam
+    prob_ham = probabilities[0].item()
+    prob_spam = probabilities[1].item()
+    
+    # Gradio expects a dictionary of {label: probability} for the Label component
+    return {"Ham (Legitimate)": prob_ham, "Spam (Malicious)": prob_spam}
+
+# 3. Create the Gradio Interface
+# We use a clean, modern interface
+demo = gr.Interface(
+    fn=predict_spam,
+    inputs=gr.Textbox(
+        lines=4, 
+        placeholder="Type an email or SMS message here...",
+        label="Message Content"
+    ),
+    outputs=gr.Label(num_top_classes=2, label="Prediction Confidence"),
+    title="🛡️ Spam Detection AI",
+    description="""
+    ### Deep Learning Project (2025)
+    This application uses a fine-tuned **DistilBERT** Transformer model to classify text messages as either Spam or Ham.
+    * Enter a message below and click Submit.
+    * **Examples of Spam:** 'URGENT! You have won a 1 week FREE membership. Call 087124006024'
+    * **Examples of Ham:** 'Hey, are we still meeting for lunch tomorrow?'
+    """,
+    examples=[
+        ["WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only."],
+        ["Hey man, just checking in. Are we still on for the movie tonight at 8?"],
+        ["URGENT: Your bank account has been locked due to suspicious activity. Please click this link to verify your identity: http://secure-login-update.com"],
+        ["I'll be about 10 minutes late, stuck in traffic."]
+    ],
+    theme=gr.themes.Soft()
+)
+
+# 4. Launch the app
+if __name__ == "__main__":
+    demo.launch()

Deep_Learning_Project/eda_script.py

@@ -0,0 +1,152 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import re
+import os
+import numpy as np
+
+# Load dataset
+df = pd.read_csv('mail_data.csv', names=['Category', 'Message'], header=None, skiprows=1)
+
+# Basic info
+print("Dataset Shape:", df.shape)
+print("\nValue Counts:\n", df['Category'].value_counts())
+print("\nMissing Values:\n", df.isnull().sum())
+
+# Add message length
+df['Length'] = df['Message'].apply(len)
+
+# Visualizations
+plt.figure(figsize=(18, 10))
+
+# 1. Class Distribution
+plt.subplot(3, 2, 1)
+sns.countplot(x='Category', data=df)
+plt.title('Class Distribution (Spam vs Ham)')
+
+# 2. Message Length Distribution
+plt.subplot(3, 2, 2)
+sns.histplot(data=df, x='Length', hue='Category', bins=50, kde=True)
+plt.title('Message Length Distribution')
+
+# Sample messages
+print("\nSample Ham:")
+print(df[df['Category'] == 'ham']['Message'].iloc[0])
+print("\nSample Spam:")
+print(df[df['Category'] == 'spam']['Message'].iloc[0])
+
+#Let's check if there are other characteristic in our text data that separates spam from ham
+
+#Word counts (how often different words are used)
+def get_word_count(messages_series: pd.Series)->tuple:
+    word_list = []
+    for message in messages_series:
+        clean_msg = re.sub("[\.|\?|,|\!]+", "", message).lower()
+        clean_msg = re.sub("\ \ +", " ", clean_msg).split(" ")
+        clean_msg = [word for word in clean_msg if word!=""]
+        word_list+=clean_msg
+
+    word_count = pd.Series(word_list).value_counts(normalize=True)
+    return word_count
+
+spam_word_count = get_word_count(df[df['Category'] == 'spam']['Message'])
+ham_word_count = get_word_count(df[df['Category'] == 'ham']['Message'])
+
+#TODO: Compare the words frequencies to find the ones used more often in spam emails compared to ham emails.
+
+words_list = list(set(spam_word_count.index)&set(ham_word_count.index))
+wordcount_distance_spam_ham = []
+for i in words_list:
+    if i in spam_word_count:
+        spam_count_i = spam_word_count[i]
+    else:
+        spam_count_i = 0
+    if i in ham_word_count:
+        ham_count_i = ham_word_count[i]
+    else:
+        ham_count_i = 0
+    wordcount_distance_spam_ham.append((ham_count_i-spam_count_i))
+wordcount_distance_spam_ham = pd.Series(wordcount_distance_spam_ham, index=words_list)
+wordcount_distance_spam_ham = wordcount_distance_spam_ham.sort_values(ascending=False)
+print("Words more often found in normal emails than spam emails")
+print(wordcount_distance_spam_ham[0:10]) #Words more often present in Ham emails
+print("words more often found in spam emails than normal emails")
+print(wordcount_distance_spam_ham[-10:]) #Words more often present in spam emails
+
+#Mail Words length (numbers of words used in the message) 
+def get_word_len(messages_series: pd.Series)->tuple:
+    word_len_list = []
+    for message in messages_series:
+        clean_msg = re.sub("[\.|\?|,|\!|0-9|]+", "", message).lower()
+        clean_msg = re.sub("\ \ +", " ", clean_msg).split(" ")
+        clean_msg = [word for word in clean_msg if word!=""]
+        word_len_list.append(len(clean_msg))
+
+    len_count = pd.Series(word_len_list)
+    return len_count
+
+df['word_len'] = get_word_len(df['Message'])
+
+plt.subplot(3, 2, 3)
+sns.histplot(data=df, x='word_len', hue='Category', bins=50, kde=True)
+plt.title('Word count Distribution')
+
+#Mail Words length (numbers of words used in the message) 
+def get_word_len2(messages_series: pd.Series)->tuple:
+    word_len_list = []
+    for message in messages_series:
+        clean_msg = re.sub("[\.|\?|,|\!|0-9]+", "", message).lower()
+        clean_msg = re.sub("\ \ +", " ", clean_msg).split(" ")
+        clean_msg = [word for word in clean_msg if word!=""]
+        total_word_length = 0
+        if len(clean_msg)>0:
+            for word in clean_msg:
+                total_word_length += len(word)
+            avg_word_len = total_word_length/len(clean_msg) 
+            word_len_list.append(avg_word_len)
+        else:
+            word_len_list.append(0)
+
+    len_count = pd.Series(word_len_list)
+    return len_count
+
+df['avg_word_len'] = get_word_len2(df['Message'])
+
+plt.subplot(3, 2, 4)
+sns.histplot(data=df, x='avg_word_len', hue='Category', bins=50, kde=True)
+plt.title('Word length Distribution')
+
+#Mail Words length (numbers of words used in the message) 
+def get_avg_sentence_len(messages_series: pd.Series)->tuple:
+    word_len_list = []
+    avg_sentence_len = []
+    for message in messages_series:
+        clean_msg = re.sub("\ \ +", " ", message).split(" ")
+        sentence_number = 1
+        sentence_finished = False
+        total_word_length = 0
+        for word in clean_msg:
+            total_word_length += 1
+            if ("." in word) or ("?" in word) or ("!" in word):
+                sentence_finished = True
+            else:
+                if sentence_finished:
+                    sentence_number+=1
+                    sentence_finished=False
+        if total_word_length > 0:
+            avg_sentence_len.append(total_word_length/sentence_number)
+    avg_sentence_len = pd.Series(avg_sentence_len)
+    return avg_sentence_len
+
+df['avg_sentence_len'] = get_avg_sentence_len(df['Message'])
+
+plt.subplot(3, 2, 5)
+sns.histplot(data=df, x='avg_sentence_len', hue='Category', bins=50, kde=True)
+plt.title('Sentence length Distribution')
+
+
+
+plt.tight_layout()
+plt.savefig('eda_plots.png')
+print("\nEDA plots saved to eda_plots.png")
+

Deep_Learning_Project/evaluate_final.py

@@ -0,0 +1,63 @@
+import pandas as pd
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix, classification_report
+from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, Trainer, TrainingArguments
+import torch
+import os
+
+# 1. Load Data
+df = pd.read_csv('mail_data.csv', names=['Category', 'Message'], header=None, skiprows=1)
+df['label'] = df['Category'].map({'ham': 0, 'spam': 1})
+
+_, test_texts, _, test_labels = train_test_split(
+    df['Message'].values.tolist(), df['label'].values.tolist(), test_size=0.2, random_state=42, stratify=df['label'].values
+)
+
+# 2. Tokenization
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+test_encodings = tokenizer(test_texts, truncation=True, padding=True, max_length=128)
+
+class EmailDataset(torch.utils.data.Dataset):
+    def __init__(self, encodings, labels):
+        self.encodings = encodings
+        self.labels = labels
+
+    def __getitem__(self, idx):
+        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+        item['labels'] = torch.tensor(self.labels[idx])
+        return item
+
+    def __len__(self):
+        return len(self.labels)
+
+test_dataset = EmailDataset(test_encodings, test_labels)
+
+# 3. Load Model from Checkpoint
+# Find the checkpoint directory
+checkpoint_dir = [d for d in os.listdir('./results') if d.startswith('checkpoint')][0]
+model_path = os.path.join('./results', checkpoint_dir)
+print(f"Loading model from {model_path}")
+
+model = DistilBertForSequenceClassification.from_pretrained(model_path)
+
+# 4. Evaluation
+trainer = Trainer(model=model)
+predictions = trainer.predict(test_dataset)
+preds = predictions.predictions.argmax(-1)
+labels = predictions.label_ids
+
+report = classification_report(labels, preds, target_names=['ham', 'spam'])
+cm = confusion_matrix(labels, preds)
+acc = accuracy_score(labels, preds)
+
+print(f"Accuracy: {acc}")
+print(report)
+
+with open('results.txt', 'w') as f:
+    f.write(f"Final Evaluation Results (from {checkpoint_dir}):\n")
+    f.write(f"Accuracy: {acc}\n")
+    f.write(f"\nClassification Report:\n{report}\n")
+    f.write(f"\nConfusion Matrix:\n{cm}\n")
+
+print("Evaluation complete. Results saved to results.txt")

Deep_Learning_Project/mail_data_test.csv

@@ -0,0 +1,9 @@
+ham,"Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat..."
+ham,Ok lar... Joking wif u oni...
+spam,Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's
+ham,U dun say so early hor... U c already then say...
+ham,"Nah I don't think he goes to usf, he lives around here though"
+spam,"FreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, £1.50 to rcv"
+ham,Even my brother is not like to speak with me. They treat me like aids patent.
+ham,As per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune
+spam,WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.

Deep_Learning_Project/project_report.md

@@ -0,0 +1,114 @@
+# Deep Learning Project: Spam Detection using Transformers
+
+**Course**: Deep Learning with Python (2025)  
+**Instructor**: Benoit Mialet  
+**Topic**: NLP - Text Classification (Spam vs Ham)  
+**Model**: DistilBERT (PyTorch / Hugging Face)
+
+---
+
+## 1. Introduction
+### 1.1 What & Why
+The objective of this project is to develop a robust deep learning model for classifying emails as either "spam" or "ham" (legitimate). Email filtering is a critical application of Natural Language Processing (NLP) that helps improve user experience and security by automatically identifying unsolicited or malicious content.
+
+### 1.2 Task Selection
+We chose the **Text Classification** task, specifically binary classification. This task is well-suited for demonstrating the power of Transfer Learning and Transformer architectures in understanding the nuances of human language.
+
+### 1.3 Relevance
+Spam detection remains a relevant challenge as spamming techniques evolve. Traditional rule-based systems often fail to capture the semantic meaning of messages. Deep learning models, particularly Transformers, can capture long-range dependencies and contextual information, leading to higher accuracy and better generalization.
+
+### 1.4 State of the Art
+Modern NLP has been revolutionized by the Transformer architecture (Vaswani et al., 2017). Models like BERT (Bidirectional Encoder Representations from Transformers) and its variants (DistilBERT, RoBERTa) have set new benchmarks in text classification by pre-training on large corpora and fine-tuning on specific tasks.
+
+---
+
+## 2. Method
+### 2.1 Overall Strategy
+Our strategy involves:
+1.  **Exploratory Data Analysis (EDA)** to understand the dataset characteristics.
+2.  **Data Preprocessing** including tokenization and padding.
+3.  **Fine-tuning a Pre-trained Model** (DistilBERT) using the Hugging Face `transformers` library and PyTorch.
+4.  **Rigorous Evaluation** using metrics like Accuracy, Precision, Recall, and F1-score.
+
+### 2.2 Dataset Description & EDA
+The dataset used is `mail_data.csv`, containing 5,572 messages labeled as 'ham' or 'spam'.
+- **Total Samples**: 5,572
+- **Ham**: 4,825 (86.6%)
+- **Spam**: 747 (13.4%)
+- **Imbalance**: The dataset is significantly imbalanced, which we addressed by using stratified splitting and monitoring the F1-score.
+
+**EDA Findings**:
+- Spam messages tend to be longer on average than ham messages.
+- Common keywords in spam include "free", "win", "winner", "call", "claim".
+- Ham messages are more conversational and vary greatly in length.
+
+### 2.3 Data Preprocessing
+- **Tokenization**: We used the `DistilBertTokenizer` to convert raw text into input IDs and attention masks.
+- **Truncation & Padding**: All sequences were padded or truncated to a maximum length of 128 tokens to ensure uniform input size for the model.
+- **Train/Test Split**: 80% training (4,457 samples) and 20% testing (1,115 samples), with stratification to maintain class proportions.
+
+### 2.4 Model Architecture
+We utilized **DistilBERT** (`distilbert-base-uncased`), a smaller, faster, and lighter version of BERT that retains 97% of its performance. It has 6 layers, 768 hidden units, and 12 attention heads, totaling approximately 66 million parameters.
+
+### 2.5 Training Setup
+- **Optimizer**: AdamW with a learning rate of 2e-5.
+- **Scheduler**: Linear warmup for 500 steps.
+- **Loss Function**: Cross-Entropy Loss.
+- **Batch Size**: 16 for training, 64 for evaluation.
+- **Epochs**: 3 (stopped early after 1 epoch due to high performance and resource constraints).
+- **Hardware**: CPU (simulated environment).
+
+---
+
+## 3. Results
+### 3.1 Performance Metrics
+The model achieved exceptional results after just one epoch of fine-tuning:
+
+| Metric | Value |
+| :--- | :--- |
+| **Accuracy** | 99.10% |
+| **Precision (Spam)** | 98.60% |
+| **Recall (Spam)** | 94.63% |
+| **F1-Score (Spam)** | 96.58% |
+
+### 3.2 Confusion Matrix
+| | Predicted Ham | Predicted Spam |
+| :--- | :---: | :---: |
+| **Actual Ham** | 964 | 2 |
+| **Actual Spam** | 8 | 141 |
+
+The model correctly identified 141 out of 149 spam messages while only misclassifying 2 legitimate messages as spam (False Positives).
+
+---
+
+## 4. Discussion
+### 4.1 Interpretation
+The high accuracy and F1-score indicate that DistilBERT is highly effective for this task. The model successfully learned the semantic patterns that distinguish spam from ham, even with a relatively small and imbalanced dataset.
+
+### 4.2 What Worked
+- **Transfer Learning**: Using a pre-trained model allowed us to achieve near-perfect results with minimal training time.
+- **Hugging Face Trainer**: Simplified the training loop and handled evaluation efficiently.
+- **Tokenization**: The subword tokenization of BERT handles out-of-vocabulary words better than traditional word-based methods.
+
+### 4.3 Limitations
+- **Dataset Size**: While sufficient for this project, a larger and more diverse dataset would be needed for a production-grade system.
+- **Class Imbalance**: Although the model performed well, the recall for spam (94.63%) is slightly lower than for ham, reflecting the imbalance.
+- **Adversarial Attacks**: Sophisticated spam might use techniques to bypass Transformer-based filters, which was not explored here.
+
+### 4.4 Future Improvements
+- **Data Augmentation**: Techniques like back-translation could help balance the dataset.
+- **Hyperparameter Tuning**: Exploring different learning rates and batch sizes.
+- **Deployment**: Creating a Gradio interface on Hugging Face Spaces for real-time testing.
+- **Model Compression**: Quantization or pruning to make the model even lighter for mobile deployment.
+
+---
+
+## 5. Conclusion
+This project successfully demonstrated the application of Deep Learning for spam detection. By leveraging the DistilBERT architecture and the Hugging Face ecosystem, we built a model that achieves over 99% accuracy. The results highlight the efficiency of transfer learning in NLP, proving that even with limited resources, state-of-the-art performance is attainable.
+
+---
+
+## 6. References
+1. Vaswani, A., et al. (2017). "Attention Is All You Need."
+2. Sanh, V., et al. (2019). "DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter."
+3. Wolf, T., et al. (2020). "Transformers: State-of-the-Art Natural Language Processing."

Deep_Learning_Project/requirements.txt

@@ -0,0 +1,9 @@
+python==3.13.12
+gradio==5.49.1
+transformers==4.57.1 
+torch==2.8.0
+numpy==2.4.2
+pandas==2.3.3
+scikit-learn==1.8.0 
+matplotlib==3.10.8
+seaborn==0.13.2 

Deep_Learning_Project/results.txt

@@ -0,0 +1,17 @@
+Final Evaluation Results:
+{'eval_loss': 0.04282991588115692, 'eval_accuracy': 0.9928251121076234, 'eval_f1': 0.972972972972973, 'eval_precision': 0.9795918367346939, 'eval_recall': 0.9664429530201343, 'eval_runtime': 42.8545, 'eval_samples_per_second': 26.018, 'eval_steps_per_second': 0.42, 'epoch': 3.0}
+
+Classification Report:
+              precision    recall  f1-score   support
+
+         ham       0.99      1.00      1.00       966
+        spam       0.98      0.97      0.97       149
+
+    accuracy                           0.99      1115
+   macro avg       0.99      0.98      0.98      1115
+weighted avg       0.99      0.99      0.99      1115
+
+
+Confusion Matrix:
+[[963   3]
+ [  5 144]]

Deep_Learning_Project/results/checkpoint-279/config.json

@@ -0,0 +1,28 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": null,
+  "dim": 768,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "problem_type": "single_label_classification",
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 30522
+}

Deep_Learning_Project/results/checkpoint-279/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:776e086853f745009f0a06b8fbcb550cc2ad70e182eb8563a473ba4e0bc9822f
+size 267832560

Deep_Learning_Project/results/checkpoint-279/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:969c762d65aebb85df619a878bc2cde125e5563dd6cb996878063145074a3333
+size 535724875

Deep_Learning_Project/results/checkpoint-279/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:826fb52114115240de98bb8ce0c3df8acf1d27dfc083ebb7276218129dcbb363
+size 14391

Deep_Learning_Project/results/checkpoint-279/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c08501cbcb6c2d9f05c9876a9e1c851c8d3b6ad5c3fae91849cbfa8c16dda076
+size 1465

Deep_Learning_Project/results/checkpoint-279/trainer_state.json

@@ -0,0 +1,235 @@
+{
+  "best_global_step": 279,
+  "best_metric": 0.04283029958605766,
+  "best_model_checkpoint": "./results\\checkpoint-279",
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 279,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035842293906810034,
+      "grad_norm": 4.070792198181152,
+      "learning_rate": 9e-07,
+      "loss": 0.6961095333099365,
+      "step": 10
+    },
+    {
+      "epoch": 0.07168458781362007,
+      "grad_norm": 3.0411295890808105,
+      "learning_rate": 1.9e-06,
+      "loss": 0.6723562717437744,
+      "step": 20
+    },
+    {
+      "epoch": 0.10752688172043011,
+      "grad_norm": 3.487802505493164,
+      "learning_rate": 2.9e-06,
+      "loss": 0.6391770362854003,
+      "step": 30
+    },
+    {
+      "epoch": 0.14336917562724014,
+      "grad_norm": 1.8233377933502197,
+      "learning_rate": 3.9e-06,
+      "loss": 0.5914762496948243,
+      "step": 40
+    },
+    {
+      "epoch": 0.17921146953405018,
+      "grad_norm": 1.4096935987472534,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.5362973213195801,
+      "step": 50
+    },
+    {
+      "epoch": 0.21505376344086022,
+      "grad_norm": 1.8130015134811401,
+      "learning_rate": 5.9e-06,
+      "loss": 0.415470027923584,
+      "step": 60
+    },
+    {
+      "epoch": 0.25089605734767023,
+      "grad_norm": 1.5945749282836914,
+      "learning_rate": 6.900000000000001e-06,
+      "loss": 0.3050834178924561,
+      "step": 70
+    },
+    {
+      "epoch": 0.2867383512544803,
+      "grad_norm": 1.1991592645645142,
+      "learning_rate": 7.9e-06,
+      "loss": 0.2292243242263794,
+      "step": 80
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 3.596205711364746,
+      "learning_rate": 8.9e-06,
+      "loss": 0.13576759099960328,
+      "step": 90
+    },
+    {
+      "epoch": 0.35842293906810035,
+      "grad_norm": 1.3767762184143066,
+      "learning_rate": 9.900000000000002e-06,
+      "loss": 0.08189120292663574,
+      "step": 100
+    },
+    {
+      "epoch": 0.3942652329749104,
+      "grad_norm": 0.7235111594200134,
+      "learning_rate": 1.09e-05,
+      "loss": 0.09285140037536621,
+      "step": 110
+    },
+    {
+      "epoch": 0.43010752688172044,
+      "grad_norm": 4.165863513946533,
+      "learning_rate": 1.19e-05,
+      "loss": 0.09866725206375122,
+      "step": 120
+    },
+    {
+      "epoch": 0.4659498207885305,
+      "grad_norm": 0.24346598982810974,
+      "learning_rate": 1.29e-05,
+      "loss": 0.02432841956615448,
+      "step": 130
+    },
+    {
+      "epoch": 0.5017921146953405,
+      "grad_norm": 0.1488434225320816,
+      "learning_rate": 1.3900000000000002e-05,
+      "loss": 0.1057550072669983,
+      "step": 140
+    },
+    {
+      "epoch": 0.5376344086021505,
+      "grad_norm": 0.16067259013652802,
+      "learning_rate": 1.49e-05,
+      "loss": 0.04291625022888183,
+      "step": 150
+    },
+    {
+      "epoch": 0.5734767025089605,
+      "grad_norm": 0.814973771572113,
+      "learning_rate": 1.59e-05,
+      "loss": 0.020908774435520174,
+      "step": 160
+    },
+    {
+      "epoch": 0.6093189964157706,
+      "grad_norm": 0.08694848418235779,
+      "learning_rate": 1.69e-05,
+      "loss": 0.06968651413917541,
+      "step": 170
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 0.08773200958967209,
+      "learning_rate": 1.79e-05,
+      "loss": 0.04258593320846558,
+      "step": 180
+    },
+    {
+      "epoch": 0.6810035842293907,
+      "grad_norm": 0.09697633981704712,
+      "learning_rate": 1.8900000000000002e-05,
+      "loss": 0.037452369928359985,
+      "step": 190
+    },
+    {
+      "epoch": 0.7168458781362007,
+      "grad_norm": 0.10511818528175354,
+      "learning_rate": 1.9900000000000003e-05,
+      "loss": 0.007081887871026993,
+      "step": 200
+    },
+    {
+      "epoch": 0.7526881720430108,
+      "grad_norm": 0.0881538987159729,
+      "learning_rate": 2.09e-05,
+      "loss": 0.008766584843397141,
+      "step": 210
+    },
+    {
+      "epoch": 0.7885304659498208,
+      "grad_norm": 0.06669408828020096,
+      "learning_rate": 2.19e-05,
+      "loss": 0.05853445529937744,
+      "step": 220
+    },
+    {
+      "epoch": 0.8243727598566308,
+      "grad_norm": 0.05606541037559509,
+      "learning_rate": 2.29e-05,
+      "loss": 0.0038808729499578477,
+      "step": 230
+    },
+    {
+      "epoch": 0.8602150537634409,
+      "grad_norm": 5.139830112457275,
+      "learning_rate": 2.39e-05,
+      "loss": 0.06458683013916015,
+      "step": 240
+    },
+    {
+      "epoch": 0.8960573476702509,
+      "grad_norm": 0.06282316893339157,
+      "learning_rate": 2.4900000000000002e-05,
+      "loss": 0.006039018556475639,
+      "step": 250
+    },
+    {
+      "epoch": 0.931899641577061,
+      "grad_norm": 7.700125694274902,
+      "learning_rate": 2.5900000000000003e-05,
+      "loss": 0.11402667760848999,
+      "step": 260
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 0.1442285031080246,
+      "learning_rate": 2.6900000000000003e-05,
+      "loss": 0.012984356284141541,
+      "step": 270
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9928251121076234,
+      "eval_f1": 0.9726027397260274,
+      "eval_loss": 0.04283029958605766,
+      "eval_precision": 0.993006993006993,
+      "eval_recall": 0.9530201342281879,
+      "eval_runtime": 42.1831,
+      "eval_samples_per_second": 26.432,
+      "eval_steps_per_second": 0.427,
+      "step": 279
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 837,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 147601798952448.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

Deep_Learning_Project/results/checkpoint-279/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fa1c04a8eee5e3e20cd86241a1b9ef2f6932b1f87ed49b5c7e1a4b6bf1a7ad0
+size 5201

Deep_Learning_Project/results/checkpoint-558/config.json

@@ -0,0 +1,28 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": null,
+  "dim": 768,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "problem_type": "single_label_classification",
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 30522
+}

Deep_Learning_Project/results/checkpoint-558/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d6801b2fc9de6ccd631b024e96ac0964d55a32c171e938c21271224dbc8da90
+size 267832560

Deep_Learning_Project/results/checkpoint-558/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2597d49d8bf7e2dd11e3084f908e6b1f989820acfae878e2ef2146d44f0dabf1
+size 535724875

Deep_Learning_Project/results/checkpoint-558/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7fd9840ce3449fb3fb17c7f4bc645548d8a9f10980ca4186834e471255c83ee
+size 14391

Deep_Learning_Project/results/checkpoint-558/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee934d9a60a571b66f851c4ecfc7a4e17ba93cd05b33a1bfee57b0d258e5611d
+size 1465

Deep_Learning_Project/results/checkpoint-558/trainer_state.json

@@ -0,0 +1,443 @@
+{
+  "best_global_step": 279,
+  "best_metric": 0.042980484664440155,
+  "best_model_checkpoint": "./results\\checkpoint-279",
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 558,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035842293906810034,
+      "grad_norm": 4.168096542358398,
+      "learning_rate": 9e-07,
+      "loss": 0.7643725395202636,
+      "step": 10
+    },
+    {
+      "epoch": 0.07168458781362007,
+      "grad_norm": 3.3584811687469482,
+      "learning_rate": 1.9e-06,
+      "loss": 0.7450664043426514,
+      "step": 20
+    },
+    {
+      "epoch": 0.10752688172043011,
+      "grad_norm": 3.753683090209961,
+      "learning_rate": 2.9e-06,
+      "loss": 0.7087752342224121,
+      "step": 30
+    },
+    {
+      "epoch": 0.14336917562724014,
+      "grad_norm": 1.9111875295639038,
+      "learning_rate": 3.9e-06,
+      "loss": 0.6540002822875977,
+      "step": 40
+    },
+    {
+      "epoch": 0.17921146953405018,
+      "grad_norm": 1.412906527519226,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.5844130516052246,
+      "step": 50
+    },
+    {
+      "epoch": 0.21505376344086022,
+      "grad_norm": 2.155179977416992,
+      "learning_rate": 5.9e-06,
+      "loss": 0.4763498783111572,
+      "step": 60
+    },
+    {
+      "epoch": 0.25089605734767023,
+      "grad_norm": 1.7028120756149292,
+      "learning_rate": 6.900000000000001e-06,
+      "loss": 0.338704252243042,
+      "step": 70
+    },
+    {
+      "epoch": 0.2867383512544803,
+      "grad_norm": 1.2706844806671143,
+      "learning_rate": 7.9e-06,
+      "loss": 0.2414313316345215,
+      "step": 80
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 1.987801194190979,
+      "learning_rate": 8.9e-06,
+      "loss": 0.13595094680786132,
+      "step": 90
+    },
+    {
+      "epoch": 0.35842293906810035,
+      "grad_norm": 1.6342447996139526,
+      "learning_rate": 9.900000000000002e-06,
+      "loss": 0.07480053305625915,
+      "step": 100
+    },
+    {
+      "epoch": 0.3942652329749104,
+      "grad_norm": 1.897525668144226,
+      "learning_rate": 1.09e-05,
+      "loss": 0.0948406457901001,
+      "step": 110
+    },
+    {
+      "epoch": 0.43010752688172044,
+      "grad_norm": 6.166868209838867,
+      "learning_rate": 1.19e-05,
+      "loss": 0.10227099657058716,
+      "step": 120
+    },
+    {
+      "epoch": 0.4659498207885305,
+      "grad_norm": 0.20176957547664642,
+      "learning_rate": 1.29e-05,
+      "loss": 0.023712341487407685,
+      "step": 130
+    },
+    {
+      "epoch": 0.5017921146953405,
+      "grad_norm": 0.1295539289712906,
+      "learning_rate": 1.3900000000000002e-05,
+      "loss": 0.0910467267036438,
+      "step": 140
+    },
+    {
+      "epoch": 0.5376344086021505,
+      "grad_norm": 0.15045320987701416,
+      "learning_rate": 1.49e-05,
+      "loss": 0.041373416781425476,
+      "step": 150
+    },
+    {
+      "epoch": 0.5734767025089605,
+      "grad_norm": 0.2986961603164673,
+      "learning_rate": 1.59e-05,
+      "loss": 0.0312202125787735,
+      "step": 160
+    },
+    {
+      "epoch": 0.6093189964157706,
+      "grad_norm": 0.0721135213971138,
+      "learning_rate": 1.69e-05,
+      "loss": 0.06126164197921753,
+      "step": 170
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 0.08172585070133209,
+      "learning_rate": 1.79e-05,
+      "loss": 0.041085737943649295,
+      "step": 180
+    },
+    {
+      "epoch": 0.6810035842293907,
+      "grad_norm": 0.09516890347003937,
+      "learning_rate": 1.8900000000000002e-05,
+      "loss": 0.04159103333950043,
+      "step": 190
+    },
+    {
+      "epoch": 0.7168458781362007,
+      "grad_norm": 0.07213900983333588,
+      "learning_rate": 1.9900000000000003e-05,
+      "loss": 0.005801299959421158,
+      "step": 200
+    },
+    {
+      "epoch": 0.7526881720430108,
+      "grad_norm": 0.053828418254852295,
+      "learning_rate": 2.09e-05,
+      "loss": 0.01323639452457428,
+      "step": 210
+    },
+    {
+      "epoch": 0.7885304659498208,
+      "grad_norm": 0.08665204793214798,
+      "learning_rate": 2.19e-05,
+      "loss": 0.06004759669303894,
+      "step": 220
+    },
+    {
+      "epoch": 0.8243727598566308,
+      "grad_norm": 0.04578182473778725,
+      "learning_rate": 2.29e-05,
+      "loss": 0.003989944979548454,
+      "step": 230
+    },
+    {
+      "epoch": 0.8602150537634409,
+      "grad_norm": 2.856813430786133,
+      "learning_rate": 2.39e-05,
+      "loss": 0.059805816411972045,
+      "step": 240
+    },
+    {
+      "epoch": 0.8960573476702509,
+      "grad_norm": 0.10952852666378021,
+      "learning_rate": 2.4900000000000002e-05,
+      "loss": 0.013767723739147187,
+      "step": 250
+    },
+    {
+      "epoch": 0.931899641577061,
+      "grad_norm": 11.293957710266113,
+      "learning_rate": 2.5900000000000003e-05,
+      "loss": 0.09849974513053894,
+      "step": 260
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 0.22327303886413574,
+      "learning_rate": 2.6900000000000003e-05,
+      "loss": 0.020736195147037506,
+      "step": 270
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9928251121076234,
+      "eval_f1": 0.9726027397260274,
+      "eval_loss": 0.042980484664440155,
+      "eval_precision": 0.993006993006993,
+      "eval_recall": 0.9530201342281879,
+      "eval_runtime": 39.0178,
+      "eval_samples_per_second": 28.577,
+      "eval_steps_per_second": 0.461,
+      "step": 279
+    },
+    {
+      "epoch": 1.003584229390681,
+      "grad_norm": 0.08417440950870514,
+      "learning_rate": 2.7900000000000004e-05,
+      "loss": 0.08314200639724731,
+      "step": 280
+    },
+    {
+      "epoch": 1.039426523297491,
+      "grad_norm": 12.878673553466797,
+      "learning_rate": 2.8899999999999998e-05,
+      "loss": 0.02687312364578247,
+      "step": 290
+    },
+    {
+      "epoch": 1.075268817204301,
+      "grad_norm": 0.16290566325187683,
+      "learning_rate": 2.9900000000000002e-05,
+      "loss": 0.029204189777374268,
+      "step": 300
+    },
+    {
+      "epoch": 1.1111111111111112,
+      "grad_norm": 0.09557493776082993,
+      "learning_rate": 3.09e-05,
+      "loss": 0.0412629634141922,
+      "step": 310
+    },
+    {
+      "epoch": 1.146953405017921,
+      "grad_norm": 0.07874058932065964,
+      "learning_rate": 3.19e-05,
+      "loss": 0.06848401427268982,
+      "step": 320
+    },
+    {
+      "epoch": 1.1827956989247312,
+      "grad_norm": 8.930388450622559,
+      "learning_rate": 3.29e-05,
+      "loss": 0.05015917420387268,
+      "step": 330
+    },
+    {
+      "epoch": 1.2186379928315412,
+      "grad_norm": 0.03418437018990517,
+      "learning_rate": 3.3900000000000004e-05,
+      "loss": 0.02710776627063751,
+      "step": 340
+    },
+    {
+      "epoch": 1.2544802867383513,
+      "grad_norm": 1.197632908821106,
+      "learning_rate": 3.49e-05,
+      "loss": 0.0023652609437704087,
+      "step": 350
+    },
+    {
+      "epoch": 1.2903225806451613,
+      "grad_norm": 9.31840991973877,
+      "learning_rate": 3.59e-05,
+      "loss": 0.050595653057098386,
+      "step": 360
+    },
+    {
+      "epoch": 1.3261648745519714,
+      "grad_norm": 4.070376873016357,
+      "learning_rate": 3.69e-05,
+      "loss": 0.0879504919052124,
+      "step": 370
+    },
+    {
+      "epoch": 1.3620071684587813,
+      "grad_norm": 0.12485872954130173,
+      "learning_rate": 3.79e-05,
+      "loss": 0.04475467205047608,
+      "step": 380
+    },
+    {
+      "epoch": 1.3978494623655915,
+      "grad_norm": 0.033023957163095474,
+      "learning_rate": 3.8900000000000004e-05,
+      "loss": 0.03315191864967346,
+      "step": 390
+    },
+    {
+      "epoch": 1.4336917562724014,
+      "grad_norm": 0.02942063845694065,
+      "learning_rate": 3.99e-05,
+      "loss": 0.023516546189785003,
+      "step": 400
+    },
+    {
+      "epoch": 1.4695340501792113,
+      "grad_norm": 0.016614006832242012,
+      "learning_rate": 4.09e-05,
+      "loss": 0.001084418874233961,
+      "step": 410
+    },
+    {
+      "epoch": 1.5053763440860215,
+      "grad_norm": 0.021838072687387466,
+      "learning_rate": 4.19e-05,
+      "loss": 0.03104546368122101,
+      "step": 420
+    },
+    {
+      "epoch": 1.5412186379928317,
+      "grad_norm": 0.07978533208370209,
+      "learning_rate": 4.29e-05,
+      "loss": 0.0017994396388530732,
+      "step": 430
+    },
+    {
+      "epoch": 1.5770609318996416,
+      "grad_norm": 0.014919363893568516,
+      "learning_rate": 4.39e-05,
+      "loss": 0.05550175905227661,
+      "step": 440
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 0.037673186510801315,
+      "learning_rate": 4.49e-05,
+      "loss": 0.09091965556144714,
+      "step": 450
+    },
+    {
+      "epoch": 1.6487455197132617,
+      "grad_norm": 17.163318634033203,
+      "learning_rate": 4.5900000000000004e-05,
+      "loss": 0.010417895019054412,
+      "step": 460
+    },
+    {
+      "epoch": 1.6845878136200718,
+      "grad_norm": 6.812087535858154,
+      "learning_rate": 4.69e-05,
+      "loss": 0.045432651042938234,
+      "step": 470
+    },
+    {
+      "epoch": 1.7204301075268817,
+      "grad_norm": 0.0256840568035841,
+      "learning_rate": 4.79e-05,
+      "loss": 0.06190497279167175,
+      "step": 480
+    },
+    {
+      "epoch": 1.7562724014336917,
+      "grad_norm": 0.08079337328672409,
+      "learning_rate": 4.89e-05,
+      "loss": 0.07536581158638,
+      "step": 490
+    },
+    {
+      "epoch": 1.7921146953405018,
+      "grad_norm": 0.020881259813904762,
+      "learning_rate": 4.99e-05,
+      "loss": 0.050325697660446166,
+      "step": 500
+    },
+    {
+      "epoch": 1.827956989247312,
+      "grad_norm": 0.011713648214936256,
+      "learning_rate": 4.8664688427299705e-05,
+      "loss": 0.020131270587444305,
+      "step": 510
+    },
+    {
+      "epoch": 1.863799283154122,
+      "grad_norm": 0.028018169105052948,
+      "learning_rate": 4.7181008902077156e-05,
+      "loss": 0.02937682271003723,
+      "step": 520
+    },
+    {
+      "epoch": 1.8996415770609318,
+      "grad_norm": 0.018673259764909744,
+      "learning_rate": 4.56973293768546e-05,
+      "loss": 0.04715012311935425,
+      "step": 530
+    },
+    {
+      "epoch": 1.935483870967742,
+      "grad_norm": 3.4265527725219727,
+      "learning_rate": 4.421364985163205e-05,
+      "loss": 0.00353032723069191,
+      "step": 540
+    },
+    {
+      "epoch": 1.971326164874552,
+      "grad_norm": 0.08214850723743439,
+      "learning_rate": 4.2729970326409497e-05,
+      "loss": 0.03979413509368897,
+      "step": 550
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9856502242152466,
+      "eval_f1": 0.9477124183006536,
+      "eval_loss": 0.06383560597896576,
+      "eval_precision": 0.9235668789808917,
+      "eval_recall": 0.9731543624161074,
+      "eval_runtime": 40.1147,
+      "eval_samples_per_second": 27.795,
+      "eval_steps_per_second": 0.449,
+      "step": 558
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 837,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 295203597904896.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

Deep_Learning_Project/results/checkpoint-558/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fa1c04a8eee5e3e20cd86241a1b9ef2f6932b1f87ed49b5c7e1a4b6bf1a7ad0
+size 5201

Deep_Learning_Project/results/checkpoint-837/config.json

@@ -0,0 +1,28 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": null,
+  "dim": 768,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "problem_type": "single_label_classification",
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 30522
+}

Deep_Learning_Project/results/checkpoint-837/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1f89da5ca6b192b1b7c42ccbe489442102b145f453959a18dc0bbac8c389997
+size 267832560

Deep_Learning_Project/results/checkpoint-837/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:630db79696e91210af33e221111cfcefe5102841bf92dff474423cfef8708af0
+size 535724875

Deep_Learning_Project/results/checkpoint-837/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d65239f408a2b9b9f7060940cb5a72607f4e554a32f1e693d531e3f14da646b
+size 14391

Deep_Learning_Project/results/checkpoint-837/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1118deb2b211462eb2b6af551d7bb98b42af89197dd44d8542729ab4409e2a1
+size 1465

Deep_Learning_Project/results/checkpoint-837/trainer_state.json

@@ -0,0 +1,651 @@
+{
+  "best_global_step": 837,
+  "best_metric": 0.04282991588115692,
+  "best_model_checkpoint": "./results\\checkpoint-837",
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 837,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035842293906810034,
+      "grad_norm": 4.168096542358398,
+      "learning_rate": 9e-07,
+      "loss": 0.7643725395202636,
+      "step": 10
+    },
+    {
+      "epoch": 0.07168458781362007,
+      "grad_norm": 3.3584811687469482,
+      "learning_rate": 1.9e-06,
+      "loss": 0.7450664043426514,
+      "step": 20
+    },
+    {
+      "epoch": 0.10752688172043011,
+      "grad_norm": 3.753683090209961,
+      "learning_rate": 2.9e-06,
+      "loss": 0.7087752342224121,
+      "step": 30
+    },
+    {
+      "epoch": 0.14336917562724014,
+      "grad_norm": 1.9111875295639038,
+      "learning_rate": 3.9e-06,
+      "loss": 0.6540002822875977,
+      "step": 40
+    },
+    {
+      "epoch": 0.17921146953405018,
+      "grad_norm": 1.412906527519226,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.5844130516052246,
+      "step": 50
+    },
+    {
+      "epoch": 0.21505376344086022,
+      "grad_norm": 2.155179977416992,
+      "learning_rate": 5.9e-06,
+      "loss": 0.4763498783111572,
+      "step": 60
+    },
+    {
+      "epoch": 0.25089605734767023,
+      "grad_norm": 1.7028120756149292,
+      "learning_rate": 6.900000000000001e-06,
+      "loss": 0.338704252243042,
+      "step": 70
+    },
+    {
+      "epoch": 0.2867383512544803,
+      "grad_norm": 1.2706844806671143,
+      "learning_rate": 7.9e-06,
+      "loss": 0.2414313316345215,
+      "step": 80
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 1.987801194190979,
+      "learning_rate": 8.9e-06,
+      "loss": 0.13595094680786132,
+      "step": 90
+    },
+    {
+      "epoch": 0.35842293906810035,
+      "grad_norm": 1.6342447996139526,
+      "learning_rate": 9.900000000000002e-06,
+      "loss": 0.07480053305625915,
+      "step": 100
+    },
+    {
+      "epoch": 0.3942652329749104,
+      "grad_norm": 1.897525668144226,
+      "learning_rate": 1.09e-05,
+      "loss": 0.0948406457901001,
+      "step": 110
+    },
+    {
+      "epoch": 0.43010752688172044,
+      "grad_norm": 6.166868209838867,
+      "learning_rate": 1.19e-05,
+      "loss": 0.10227099657058716,
+      "step": 120
+    },
+    {
+      "epoch": 0.4659498207885305,
+      "grad_norm": 0.20176957547664642,
+      "learning_rate": 1.29e-05,
+      "loss": 0.023712341487407685,
+      "step": 130
+    },
+    {
+      "epoch": 0.5017921146953405,
+      "grad_norm": 0.1295539289712906,
+      "learning_rate": 1.3900000000000002e-05,
+      "loss": 0.0910467267036438,
+      "step": 140
+    },
+    {
+      "epoch": 0.5376344086021505,
+      "grad_norm": 0.15045320987701416,
+      "learning_rate": 1.49e-05,
+      "loss": 0.041373416781425476,
+      "step": 150
+    },
+    {
+      "epoch": 0.5734767025089605,
+      "grad_norm": 0.2986961603164673,
+      "learning_rate": 1.59e-05,
+      "loss": 0.0312202125787735,
+      "step": 160
+    },
+    {
+      "epoch": 0.6093189964157706,
+      "grad_norm": 0.0721135213971138,
+      "learning_rate": 1.69e-05,
+      "loss": 0.06126164197921753,
+      "step": 170
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 0.08172585070133209,
+      "learning_rate": 1.79e-05,
+      "loss": 0.041085737943649295,
+      "step": 180
+    },
+    {
+      "epoch": 0.6810035842293907,
+      "grad_norm": 0.09516890347003937,
+      "learning_rate": 1.8900000000000002e-05,
+      "loss": 0.04159103333950043,
+      "step": 190
+    },
+    {
+      "epoch": 0.7168458781362007,
+      "grad_norm": 0.07213900983333588,
+      "learning_rate": 1.9900000000000003e-05,
+      "loss": 0.005801299959421158,
+      "step": 200
+    },
+    {
+      "epoch": 0.7526881720430108,
+      "grad_norm": 0.053828418254852295,
+      "learning_rate": 2.09e-05,
+      "loss": 0.01323639452457428,
+      "step": 210
+    },
+    {
+      "epoch": 0.7885304659498208,
+      "grad_norm": 0.08665204793214798,
+      "learning_rate": 2.19e-05,
+      "loss": 0.06004759669303894,
+      "step": 220
+    },
+    {
+      "epoch": 0.8243727598566308,
+      "grad_norm": 0.04578182473778725,
+      "learning_rate": 2.29e-05,
+      "loss": 0.003989944979548454,
+      "step": 230
+    },
+    {
+      "epoch": 0.8602150537634409,
+      "grad_norm": 2.856813430786133,
+      "learning_rate": 2.39e-05,
+      "loss": 0.059805816411972045,
+      "step": 240
+    },
+    {
+      "epoch": 0.8960573476702509,
+      "grad_norm": 0.10952852666378021,
+      "learning_rate": 2.4900000000000002e-05,
+      "loss": 0.013767723739147187,
+      "step": 250
+    },
+    {
+      "epoch": 0.931899641577061,
+      "grad_norm": 11.293957710266113,
+      "learning_rate": 2.5900000000000003e-05,
+      "loss": 0.09849974513053894,
+      "step": 260
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 0.22327303886413574,
+      "learning_rate": 2.6900000000000003e-05,
+      "loss": 0.020736195147037506,
+      "step": 270
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9928251121076234,
+      "eval_f1": 0.9726027397260274,
+      "eval_loss": 0.042980484664440155,
+      "eval_precision": 0.993006993006993,
+      "eval_recall": 0.9530201342281879,
+      "eval_runtime": 39.0178,
+      "eval_samples_per_second": 28.577,
+      "eval_steps_per_second": 0.461,
+      "step": 279
+    },
+    {
+      "epoch": 1.003584229390681,
+      "grad_norm": 0.08417440950870514,
+      "learning_rate": 2.7900000000000004e-05,
+      "loss": 0.08314200639724731,
+      "step": 280
+    },
+    {
+      "epoch": 1.039426523297491,
+      "grad_norm": 12.878673553466797,
+      "learning_rate": 2.8899999999999998e-05,
+      "loss": 0.02687312364578247,
+      "step": 290
+    },
+    {
+      "epoch": 1.075268817204301,
+      "grad_norm": 0.16290566325187683,
+      "learning_rate": 2.9900000000000002e-05,
+      "loss": 0.029204189777374268,
+      "step": 300
+    },
+    {
+      "epoch": 1.1111111111111112,
+      "grad_norm": 0.09557493776082993,
+      "learning_rate": 3.09e-05,
+      "loss": 0.0412629634141922,
+      "step": 310
+    },
+    {
+      "epoch": 1.146953405017921,
+      "grad_norm": 0.07874058932065964,
+      "learning_rate": 3.19e-05,
+      "loss": 0.06848401427268982,
+      "step": 320
+    },
+    {
+      "epoch": 1.1827956989247312,
+      "grad_norm": 8.930388450622559,
+      "learning_rate": 3.29e-05,
+      "loss": 0.05015917420387268,
+      "step": 330
+    },
+    {
+      "epoch": 1.2186379928315412,
+      "grad_norm": 0.03418437018990517,
+      "learning_rate": 3.3900000000000004e-05,
+      "loss": 0.02710776627063751,
+      "step": 340
+    },
+    {
+      "epoch": 1.2544802867383513,
+      "grad_norm": 1.197632908821106,
+      "learning_rate": 3.49e-05,
+      "loss": 0.0023652609437704087,
+      "step": 350
+    },
+    {
+      "epoch": 1.2903225806451613,
+      "grad_norm": 9.31840991973877,
+      "learning_rate": 3.59e-05,
+      "loss": 0.050595653057098386,
+      "step": 360
+    },
+    {
+      "epoch": 1.3261648745519714,
+      "grad_norm": 4.070376873016357,
+      "learning_rate": 3.69e-05,
+      "loss": 0.0879504919052124,
+      "step": 370
+    },
+    {
+      "epoch": 1.3620071684587813,
+      "grad_norm": 0.12485872954130173,
+      "learning_rate": 3.79e-05,
+      "loss": 0.04475467205047608,
+      "step": 380
+    },
+    {
+      "epoch": 1.3978494623655915,
+      "grad_norm": 0.033023957163095474,
+      "learning_rate": 3.8900000000000004e-05,
+      "loss": 0.03315191864967346,
+      "step": 390
+    },
+    {
+      "epoch": 1.4336917562724014,
+      "grad_norm": 0.02942063845694065,
+      "learning_rate": 3.99e-05,
+      "loss": 0.023516546189785003,
+      "step": 400
+    },
+    {
+      "epoch": 1.4695340501792113,
+      "grad_norm": 0.016614006832242012,
+      "learning_rate": 4.09e-05,
+      "loss": 0.001084418874233961,
+      "step": 410
+    },
+    {
+      "epoch": 1.5053763440860215,
+      "grad_norm": 0.021838072687387466,
+      "learning_rate": 4.19e-05,
+      "loss": 0.03104546368122101,
+      "step": 420
+    },
+    {
+      "epoch": 1.5412186379928317,
+      "grad_norm": 0.07978533208370209,
+      "learning_rate": 4.29e-05,
+      "loss": 0.0017994396388530732,
+      "step": 430
+    },
+    {
+      "epoch": 1.5770609318996416,
+      "grad_norm": 0.014919363893568516,
+      "learning_rate": 4.39e-05,
+      "loss": 0.05550175905227661,
+      "step": 440
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 0.037673186510801315,
+      "learning_rate": 4.49e-05,
+      "loss": 0.09091965556144714,
+      "step": 450
+    },
+    {
+      "epoch": 1.6487455197132617,
+      "grad_norm": 17.163318634033203,
+      "learning_rate": 4.5900000000000004e-05,
+      "loss": 0.010417895019054412,
+      "step": 460
+    },
+    {
+      "epoch": 1.6845878136200718,
+      "grad_norm": 6.812087535858154,
+      "learning_rate": 4.69e-05,
+      "loss": 0.045432651042938234,
+      "step": 470
+    },
+    {
+      "epoch": 1.7204301075268817,
+      "grad_norm": 0.0256840568035841,
+      "learning_rate": 4.79e-05,
+      "loss": 0.06190497279167175,
+      "step": 480
+    },
+    {
+      "epoch": 1.7562724014336917,
+      "grad_norm": 0.08079337328672409,
+      "learning_rate": 4.89e-05,
+      "loss": 0.07536581158638,
+      "step": 490
+    },
+    {
+      "epoch": 1.7921146953405018,
+      "grad_norm": 0.020881259813904762,
+      "learning_rate": 4.99e-05,
+      "loss": 0.050325697660446166,
+      "step": 500
+    },
+    {
+      "epoch": 1.827956989247312,
+      "grad_norm": 0.011713648214936256,
+      "learning_rate": 4.8664688427299705e-05,
+      "loss": 0.020131270587444305,
+      "step": 510
+    },
+    {
+      "epoch": 1.863799283154122,
+      "grad_norm": 0.028018169105052948,
+      "learning_rate": 4.7181008902077156e-05,
+      "loss": 0.02937682271003723,
+      "step": 520
+    },
+    {
+      "epoch": 1.8996415770609318,
+      "grad_norm": 0.018673259764909744,
+      "learning_rate": 4.56973293768546e-05,
+      "loss": 0.04715012311935425,
+      "step": 530
+    },
+    {
+      "epoch": 1.935483870967742,
+      "grad_norm": 3.4265527725219727,
+      "learning_rate": 4.421364985163205e-05,
+      "loss": 0.00353032723069191,
+      "step": 540
+    },
+    {
+      "epoch": 1.971326164874552,
+      "grad_norm": 0.08214850723743439,
+      "learning_rate": 4.2729970326409497e-05,
+      "loss": 0.03979413509368897,
+      "step": 550
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9856502242152466,
+      "eval_f1": 0.9477124183006536,
+      "eval_loss": 0.06383560597896576,
+      "eval_precision": 0.9235668789808917,
+      "eval_recall": 0.9731543624161074,
+      "eval_runtime": 40.1147,
+      "eval_samples_per_second": 27.795,
+      "eval_steps_per_second": 0.449,
+      "step": 558
+    },
+    {
+      "epoch": 2.007168458781362,
+      "grad_norm": 0.01592390425503254,
+      "learning_rate": 4.124629080118694e-05,
+      "loss": 0.010815779119729996,
+      "step": 560
+    },
+    {
+      "epoch": 2.043010752688172,
+      "grad_norm": 0.011524681001901627,
+      "learning_rate": 3.976261127596439e-05,
+      "loss": 0.0023574797436594964,
+      "step": 570
+    },
+    {
+      "epoch": 2.078853046594982,
+      "grad_norm": 0.009996078908443451,
+      "learning_rate": 3.8278931750741844e-05,
+      "loss": 0.0014362875372171402,
+      "step": 580
+    },
+    {
+      "epoch": 2.1146953405017923,
+      "grad_norm": 0.01068540196865797,
+      "learning_rate": 3.679525222551929e-05,
+      "loss": 0.0005183162167668343,
+      "step": 590
+    },
+    {
+      "epoch": 2.150537634408602,
+      "grad_norm": 0.007895471528172493,
+      "learning_rate": 3.531157270029673e-05,
+      "loss": 0.000452942680567503,
+      "step": 600
+    },
+    {
+      "epoch": 2.186379928315412,
+      "grad_norm": 0.011167576536536217,
+      "learning_rate": 3.382789317507419e-05,
+      "loss": 0.04565889239311218,
+      "step": 610
+    },
+    {
+      "epoch": 2.2222222222222223,
+      "grad_norm": 0.021645231172442436,
+      "learning_rate": 3.2344213649851636e-05,
+      "loss": 0.04359175860881805,
+      "step": 620
+    },
+    {
+      "epoch": 2.258064516129032,
+      "grad_norm": 0.020626947283744812,
+      "learning_rate": 3.086053412462908e-05,
+      "loss": 0.0012028517201542854,
+      "step": 630
+    },
+    {
+      "epoch": 2.293906810035842,
+      "grad_norm": 2.1626150608062744,
+      "learning_rate": 2.937685459940653e-05,
+      "loss": 0.04995043277740478,
+      "step": 640
+    },
+    {
+      "epoch": 2.3297491039426523,
+      "grad_norm": 0.024094557389616966,
+      "learning_rate": 2.789317507418398e-05,
+      "loss": 0.0011813377030193805,
+      "step": 650
+    },
+    {
+      "epoch": 2.3655913978494625,
+      "grad_norm": 0.024276426061987877,
+      "learning_rate": 2.6409495548961428e-05,
+      "loss": 0.000971246138215065,
+      "step": 660
+    },
+    {
+      "epoch": 2.4014336917562726,
+      "grad_norm": 0.013462797738611698,
+      "learning_rate": 2.4925816023738872e-05,
+      "loss": 0.001258693542331457,
+      "step": 670
+    },
+    {
+      "epoch": 2.4372759856630823,
+      "grad_norm": 0.011892343871295452,
+      "learning_rate": 2.344213649851632e-05,
+      "loss": 0.000927103403955698,
+      "step": 680
+    },
+    {
+      "epoch": 2.4731182795698925,
+      "grad_norm": 0.020849108695983887,
+      "learning_rate": 2.195845697329377e-05,
+      "loss": 0.048611697554588315,
+      "step": 690
+    },
+    {
+      "epoch": 2.5089605734767026,
+      "grad_norm": 0.027146149426698685,
+      "learning_rate": 2.0474777448071216e-05,
+      "loss": 0.0010239629074931145,
+      "step": 700
+    },
+    {
+      "epoch": 2.5448028673835124,
+      "grad_norm": 0.02077840454876423,
+      "learning_rate": 1.8991097922848668e-05,
+      "loss": 0.011852725595235824,
+      "step": 710
+    },
+    {
+      "epoch": 2.5806451612903225,
+      "grad_norm": 0.013969900086522102,
+      "learning_rate": 1.7507418397626112e-05,
+      "loss": 0.0008809153921902179,
+      "step": 720
+    },
+    {
+      "epoch": 2.6164874551971327,
+      "grad_norm": 0.01932111196219921,
+      "learning_rate": 1.6023738872403564e-05,
+      "loss": 0.0008839547634124755,
+      "step": 730
+    },
+    {
+      "epoch": 2.652329749103943,
+      "grad_norm": 0.011346139945089817,
+      "learning_rate": 1.454005934718101e-05,
+      "loss": 0.0007063972763717175,
+      "step": 740
+    },
+    {
+      "epoch": 2.688172043010753,
+      "grad_norm": 0.02365295961499214,
+      "learning_rate": 1.3056379821958458e-05,
+      "loss": 0.03284582793712616,
+      "step": 750
+    },
+    {
+      "epoch": 2.7240143369175627,
+      "grad_norm": 0.019500477239489555,
+      "learning_rate": 1.1572700296735906e-05,
+      "loss": 0.009724142402410508,
+      "step": 760
+    },
+    {
+      "epoch": 2.759856630824373,
+      "grad_norm": 0.013806493952870369,
+      "learning_rate": 1.0089020771513354e-05,
+      "loss": 0.000749332644045353,
+      "step": 770
+    },
+    {
+      "epoch": 2.795698924731183,
+      "grad_norm": 0.04070394113659859,
+      "learning_rate": 8.605341246290802e-06,
+      "loss": 0.0005624637007713318,
+      "step": 780
+    },
+    {
+      "epoch": 2.8315412186379927,
+      "grad_norm": 0.014284063130617142,
+      "learning_rate": 7.12166172106825e-06,
+      "loss": 0.0005397583357989788,
+      "step": 790
+    },
+    {
+      "epoch": 2.867383512544803,
+      "grad_norm": 0.01228868868201971,
+      "learning_rate": 5.637982195845697e-06,
+      "loss": 0.0026672758162021638,
+      "step": 800
+    },
+    {
+      "epoch": 2.903225806451613,
+      "grad_norm": 0.007240036968141794,
+      "learning_rate": 4.154302670623145e-06,
+      "loss": 0.0004312596283853054,
+      "step": 810
+    },
+    {
+      "epoch": 2.9390681003584227,
+      "grad_norm": 0.010902538895606995,
+      "learning_rate": 2.6706231454005935e-06,
+      "loss": 0.0004907793365418911,
+      "step": 820
+    },
+    {
+      "epoch": 2.974910394265233,
+      "grad_norm": 0.011084447614848614,
+      "learning_rate": 1.1869436201780417e-06,
+      "loss": 0.01800166815519333,
+      "step": 830
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9928251121076234,
+      "eval_f1": 0.972972972972973,
+      "eval_loss": 0.04282991588115692,
+      "eval_precision": 0.9795918367346939,
+      "eval_recall": 0.9664429530201343,
+      "eval_runtime": 42.4854,
+      "eval_samples_per_second": 26.244,
+      "eval_steps_per_second": 0.424,
+      "step": 837
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 837,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 442805396857344.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

Deep_Learning_Project/results/checkpoint-837/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fa1c04a8eee5e3e20cd86241a1b9ef2f6932b1f87ed49b5c7e1a4b6bf1a7ad0
+size 5201

Deep_Learning_Project/save_tokenizer.py

@@ -0,0 +1,5 @@
+from transformers import DistilBertTokenizer
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+tokenizer.save_pretrained('saved_model')
+
+print("Tokenizer saved to saved_model")

Deep_Learning_Project/saved_model/config.json

@@ -0,0 +1,28 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": null,
+  "dim": 768,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "problem_type": "single_label_classification",
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 30522
+}

Deep_Learning_Project/saved_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c07d77d4bad466798bc928c36a02d91cadbef0c2047def7e445928f1825d959
+size 267832560

Deep_Learning_Project/saved_model/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b616fb2661c2ad4dbc6c0ee5c4c1caefd532989e8c399005744f5442e4116ed
+size 535724875

Deep_Learning_Project/saved_model/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c0f6bef14c378cf6dadcce9566f94c89a1de5a87f6dab300dc2cfb3620974e5
+size 14455

Deep_Learning_Project/saved_model/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd612b91f9c87518d4d69e324f23842dc16c0f0824a97f416850db8d65d6175
+size 1465

Deep_Learning_Project/saved_model/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "is_local": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "unk_token": "[UNK]"
+}

Deep_Learning_Project/saved_model/trainer_state.json

@@ -0,0 +1,235 @@
+{
+  "best_global_step": 279,
+  "best_metric": 0.04009857773780823,
+  "best_model_checkpoint": "./results/checkpoint-279",
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 279,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.035842293906810034,
+      "grad_norm": 4.057015895843506,
+      "learning_rate": 9e-07,
+      "loss": 0.6395920753479004,
+      "step": 10
+    },
+    {
+      "epoch": 0.07168458781362007,
+      "grad_norm": 3.033071279525757,
+      "learning_rate": 1.9e-06,
+      "loss": 0.6280024528503418,
+      "step": 20
+    },
+    {
+      "epoch": 0.10752688172043011,
+      "grad_norm": 3.566011905670166,
+      "learning_rate": 2.9e-06,
+      "loss": 0.6011054992675782,
+      "step": 30
+    },
+    {
+      "epoch": 0.14336917562724014,
+      "grad_norm": 1.8344322443008423,
+      "learning_rate": 3.9e-06,
+      "loss": 0.5574448108673096,
+      "step": 40
+    },
+    {
+      "epoch": 0.17921146953405018,
+      "grad_norm": 1.37172269821167,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.5156032562255859,
+      "step": 50
+    },
+    {
+      "epoch": 0.21505376344086022,
+      "grad_norm": 1.646365761756897,
+      "learning_rate": 5.9e-06,
+      "loss": 0.40119266510009766,
+      "step": 60
+    },
+    {
+      "epoch": 0.25089605734767023,
+      "grad_norm": 1.6370402574539185,
+      "learning_rate": 6.900000000000001e-06,
+      "loss": 0.2945317268371582,
+      "step": 70
+    },
+    {
+      "epoch": 0.2867383512544803,
+      "grad_norm": 1.153666377067566,
+      "learning_rate": 7.9e-06,
+      "loss": 0.2097024440765381,
+      "step": 80
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 4.979272365570068,
+      "learning_rate": 8.9e-06,
+      "loss": 0.1338476538658142,
+      "step": 90
+    },
+    {
+      "epoch": 0.35842293906810035,
+      "grad_norm": 1.4230221509933472,
+      "learning_rate": 9.900000000000002e-06,
+      "loss": 0.07898266315460205,
+      "step": 100
+    },
+    {
+      "epoch": 0.3942652329749104,
+      "grad_norm": 0.4248649477958679,
+      "learning_rate": 1.09e-05,
+      "loss": 0.09649826884269715,
+      "step": 110
+    },
+    {
+      "epoch": 0.43010752688172044,
+      "grad_norm": 5.597842216491699,
+      "learning_rate": 1.19e-05,
+      "loss": 0.10944361686706543,
+      "step": 120
+    },
+    {
+      "epoch": 0.4659498207885305,
+      "grad_norm": 0.2298722118139267,
+      "learning_rate": 1.29e-05,
+      "loss": 0.02345200479030609,
+      "step": 130
+    },
+    {
+      "epoch": 0.5017921146953405,
+      "grad_norm": 0.16554060578346252,
+      "learning_rate": 1.3900000000000002e-05,
+      "loss": 0.10124502182006836,
+      "step": 140
+    },
+    {
+      "epoch": 0.5376344086021505,
+      "grad_norm": 0.20511625707149506,
+      "learning_rate": 1.49e-05,
+      "loss": 0.03878947794437408,
+      "step": 150
+    },
+    {
+      "epoch": 0.5734767025089605,
+      "grad_norm": 0.22746935486793518,
+      "learning_rate": 1.59e-05,
+      "loss": 0.015351028740406036,
+      "step": 160
+    },
+    {
+      "epoch": 0.6093189964157706,
+      "grad_norm": 0.09672143310308456,
+      "learning_rate": 1.69e-05,
+      "loss": 0.08337704539299011,
+      "step": 170
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 0.09403681755065918,
+      "learning_rate": 1.79e-05,
+      "loss": 0.04203044176101685,
+      "step": 180
+    },
+    {
+      "epoch": 0.6810035842293907,
+      "grad_norm": 0.10677973181009293,
+      "learning_rate": 1.8900000000000002e-05,
+      "loss": 0.0367545485496521,
+      "step": 190
+    },
+    {
+      "epoch": 0.7168458781362007,
+      "grad_norm": 0.09498517960309982,
+      "learning_rate": 1.9900000000000003e-05,
+      "loss": 0.006626572459936142,
+      "step": 200
+    },
+    {
+      "epoch": 0.7526881720430108,
+      "grad_norm": 0.058950286358594894,
+      "learning_rate": 2.09e-05,
+      "loss": 0.004921931400895118,
+      "step": 210
+    },
+    {
+      "epoch": 0.7885304659498208,
+      "grad_norm": 0.05829274281859398,
+      "learning_rate": 2.19e-05,
+      "loss": 0.06366445422172547,
+      "step": 220
+    },
+    {
+      "epoch": 0.8243727598566308,
+      "grad_norm": 0.0513160303235054,
+      "learning_rate": 2.29e-05,
+      "loss": 0.0034723080694675445,
+      "step": 230
+    },
+    {
+      "epoch": 0.8602150537634409,
+      "grad_norm": 8.491065979003906,
+      "learning_rate": 2.39e-05,
+      "loss": 0.06077749729156494,
+      "step": 240
+    },
+    {
+      "epoch": 0.8960573476702509,
+      "grad_norm": 0.05915055796504021,
+      "learning_rate": 2.4900000000000002e-05,
+      "loss": 0.009128168970346451,
+      "step": 250
+    },
+    {
+      "epoch": 0.931899641577061,
+      "grad_norm": 10.863852500915527,
+      "learning_rate": 2.5900000000000003e-05,
+      "loss": 0.09072564840316773,
+      "step": 260
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 0.42785295844078064,
+      "learning_rate": 2.6900000000000003e-05,
+      "loss": 0.03498620092868805,
+      "step": 270
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9910313901345291,
+      "eval_f1": 0.9657534246575342,
+      "eval_loss": 0.04009857773780823,
+      "eval_precision": 0.986013986013986,
+      "eval_recall": 0.9463087248322147,
+      "eval_runtime": 32.5434,
+      "eval_samples_per_second": 34.262,
+      "eval_steps_per_second": 0.553,
+      "step": 279
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 837,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 147601798952448.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

Deep_Learning_Project/saved_model/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab8b54e46b16083bb42102640d659199bfe96644c6fe4869e1135f8028f75f5c
+size 5201

Deep_Learning_Project/train_model.py

@@ -0,0 +1,152 @@
+import pandas as pd
+import torch
+from torch.utils.data import Dataset, DataLoader
+from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, get_linear_schedule_with_warmup
+from torch.optim import AdamW
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+import numpy as np
+import time
+import os
+
+# 1. Load and Preprocess Data
+df = pd.read_csv('mail_data.csv', names=['Category', 'Message'], header=None, skiprows=1)
+df['label'] = df['Category'].map({'ham': 0, 'spam': 1})
+
+train_texts, test_texts, train_labels, test_labels = train_test_split(
+    df['Message'].values, df['label'].values, test_size=0.2, random_state=42, stratify=df['label'].values
+)
+
+# 2. Dataset Class
+class EmailDataset(Dataset):
+    def __init__(self, texts, labels, tokenizer, max_len=128):
+        self.texts = texts
+        self.labels = labels
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, item):
+        text = str(self.texts[item])
+        label = self.labels[item]
+        encoding = self.tokenizer._encode_plus(
+            text,
+            add_special_tokens=True,
+            max_length=self.max_len,
+            return_token_type_ids=False,
+            padding='max_length',
+            truncation=True,
+            return_attention_mask=True,
+            return_tensors='pt',
+        )
+        return {
+            'text': text,
+            'input_ids': encoding['input_ids'].flatten(),
+            'attention_mask': encoding['attention_mask'].flatten(),
+            'labels': torch.tensor(label, dtype=torch.long)
+        }
+
+# 3. Setup Training
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"Using device: {device}")
+
+PRE_TRAINED_MODEL_NAME = 'distilbert-base-uncased'
+tokenizer = DistilBertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)
+
+train_data_loader = DataLoader(EmailDataset(train_texts, train_labels, tokenizer), batch_size=16, shuffle=True)
+test_data_loader = DataLoader(EmailDataset(test_texts, test_labels, tokenizer), batch_size=16, shuffle=False)
+
+model = DistilBertForSequenceClassification.from_pretrained(PRE_TRAINED_MODEL_NAME, num_labels=2)
+model = model.to(device)
+
+EPOCHS = 3
+optimizer = AdamW(model.parameters(), lr=2e-5)
+total_steps = len(train_data_loader) * EPOCHS
+scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)
+loss_fn = torch.nn.CrossEntropyLoss().to(device)
+
+# 4. Training Loop
+def train_epoch(model, data_loader, loss_fn, optimizer, device, scheduler, n_examples):
+    model = model.train()
+    losses = []
+    correct_predictions = 0
+    for d in data_loader:
+        input_ids = d["input_ids"].to(device)
+        attention_mask = d["attention_mask"].to(device)
+        labels = d["labels"].to(device)
+        outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels)
+        loss = outputs.loss
+        logits = outputs.logits
+        _, preds = torch.max(logits, dim=1)
+        correct_predictions += torch.sum(preds == labels)
+        losses.append(loss.item())
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+        optimizer.step()
+        scheduler.step()
+        optimizer.zero_grad()
+    return correct_predictions.double() / n_examples, np.mean(losses)
+
+def eval_model(model, data_loader, loss_fn, device, n_examples):
+    model = model.eval()
+    losses = []
+    correct_predictions = 0
+    with torch.no_grad():
+        for d in data_loader:
+            input_ids = d["input_ids"].to(device)
+            attention_mask = d["attention_mask"].to(device)
+            labels = d["labels"].to(device)
+            outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels)
+            loss = outputs.loss
+            logits = outputs.logits
+            _, preds = torch.max(logits, dim=1)
+            correct_predictions += torch.sum(preds == labels)
+            losses.append(loss.item())
+    return correct_predictions.double() / n_examples, np.mean(losses)
+
+print("Starting training...")
+for epoch in range(EPOCHS):
+    print(f'Epoch {epoch + 1}/{EPOCHS}')
+    train_acc, train_loss = train_epoch(model, train_data_loader, loss_fn, optimizer, device, scheduler, len(train_texts))
+    print(f'Train loss {train_loss} accuracy {train_acc}')
+    val_acc, val_loss = eval_model(model, test_data_loader, loss_fn, device, len(test_texts))
+    print(f'Val   loss {val_loss} accuracy {val_acc}')
+
+# 5. Final Evaluation
+def get_predictions(model, data_loader):
+    model = model.eval()
+    messages = []
+    predictions = []
+    prediction_probs = []
+    real_values = []
+    with torch.no_grad():
+        for d in data_loader:
+            texts = d["text"]
+            input_ids = d["input_ids"].to(device)
+            attention_mask = d["attention_mask"].to(device)
+            labels = d["labels"].to(device)
+            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
+            logits = outputs.logits
+            _, preds = torch.max(logits, dim=1)
+            messages.extend(texts)
+            predictions.extend(preds)
+            prediction_probs.extend(logits)
+            real_values.extend(labels)
+    predictions = torch.stack(predictions).cpu()
+    real_values = torch.stack(real_values).cpu()
+    return messages, predictions, real_values
+
+y_review_texts, y_pred, y_test = get_predictions(model, test_data_loader)
+print("\nClassification Report:\n", classification_report(y_test, y_pred, target_names=['ham', 'spam']))
+
+# Save results for report
+with open('results.txt', 'w') as f:
+    f.write(f"Accuracy: {accuracy_score(y_test, y_pred)}\n")
+    f.write("\nClassification Report:\n")
+    f.write(classification_report(y_test, y_pred, target_names=['ham', 'spam']))
+    f.write("\nConfusion Matrix:\n")
+    f.write(str(confusion_matrix(y_test, y_pred)))
+
+print("Training complete. Results saved to results.txt")

Deep_Learning_Project/train_model_hf.py

@@ -0,0 +1,99 @@
+import pandas as pd
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix
+from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, Trainer, TrainingArguments
+import torch
+
+# 1. Load and Preprocess Data
+df = pd.read_csv('mail_data.csv', names=['Category', 'Message'], header=None, skiprows=1)
+df['label'] = df['Category'].map({'ham': 0, 'spam': 1})
+
+train_texts, test_texts, train_labels, test_labels = train_test_split(
+    df['Message'].values.tolist(), df['label'].values.tolist(), test_size=0.2, random_state=42, stratify=df['label'].values
+)
+
+# 2. Tokenization
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+
+train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=128)
+test_encodings = tokenizer(test_texts, truncation=True, padding=True, max_length=128)
+
+class EmailDataset(torch.utils.data.Dataset):
+    def __init__(self, encodings, labels):
+        self.encodings = encodings
+        self.labels = labels
+
+    def __getitem__(self, idx):
+        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+        item['labels'] = torch.tensor(self.labels[idx])
+        return item
+
+    def __len__(self):
+        return len(self.labels)
+
+train_dataset = EmailDataset(train_encodings, train_labels)
+test_dataset = EmailDataset(test_encodings, test_labels)
+
+# 3. Model and Metrics
+model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=2)
+
+def compute_metrics(pred):
+    labels = pred.label_ids
+    preds = pred.predictions.argmax(-1)
+    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')
+    acc = accuracy_score(labels, preds)
+    return {
+        'accuracy': acc,
+        'f1': f1,
+        'precision': precision,
+        'recall': recall
+    }
+
+# 4. Training Arguments
+training_args = TrainingArguments(
+    output_dir='./results',
+    num_train_epochs=3,
+    per_device_train_batch_size=16,
+    per_device_eval_batch_size=64,
+    warmup_steps=500,
+    weight_decay=0.01,
+    logging_dir='./logs',
+    logging_steps=10,
+    eval_strategy="epoch",
+    save_strategy="epoch",
+    load_best_model_at_end=True,
+)
+
+# 5. Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=test_dataset,
+    compute_metrics=compute_metrics,
+)
+
+print("Starting training with HF Trainer...")
+trainer.train()
+
+# 6. Evaluation
+print("Evaluating...")
+eval_results = trainer.evaluate()
+print(eval_results)
+
+# Final predictions for detailed report
+predictions = trainer.predict(test_dataset)
+preds = predictions.predictions.argmax(-1)
+labels = predictions.label_ids
+
+from sklearn.metrics import classification_report
+report = classification_report(labels, preds, target_names=['ham', 'spam'])
+cm = confusion_matrix(labels, preds)
+
+with open('results.txt', 'w') as f:
+    f.write(f"Final Evaluation Results:\n{eval_results}\n")
+    f.write(f"\nClassification Report:\n{report}\n")
+    f.write(f"\nConfusion Matrix:\n{cm}\n")
+
+print("Training complete. Results saved to results.txt")