Create utilis.py

utilis.py

@@ -0,0 +1,59 @@
+import os
+from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
+from datasets import load_dataset
+
+def load_model_and_tokenizer(model_name):
+    """
+    Load the model and tokenizer.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name)
+    return model, tokenizer
+
+def load_and_tokenize_dataset(dataset_name, tokenizer, max_length=512):
+    """
+    Load and tokenize the dataset.
+    """
+    dataset = load_dataset(dataset_name)
+
+    def tokenize_function(examples):
+        return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=max_length)
+
+    tokenized_datasets = dataset.map(tokenize_function, batched=True)
+    return tokenized_datasets
+
+def setup_training_args(output_dir="./results", per_device_train_batch_size=2, per_device_eval_batch_size=2,
+                        gradient_accumulation_steps=8, num_train_epochs=3, learning_rate=5e-5, weight_decay=0.01,
+                        warmup_steps=500, logging_steps=100, fp16=True):
+    """
+    Set up training arguments.
+    """
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        evaluation_strategy="epoch",
+        per_device_train_batch_size=per_device_train_batch_size,
+        per_device_eval_batch_size=per_device_eval_batch_size,
+        gradient_accumulation_steps=gradient_accumulation_steps,
+        num_train_epochs=num_train_epochs,
+        save_strategy="epoch",
+        save_total_limit=2,
+        logging_dir="./logs",
+        logging_steps=logging_steps,
+        report_to="none",
+        fp16=fp16,
+        learning_rate=learning_rate,
+        weight_decay=weight_decay,
+        warmup_steps=warmup_steps,
+        dataloader_num_workers=4,
+        push_to_hub=False
+    )
+    return training_args
+
+def save_model_and_tokenizer(model, tokenizer, save_dir):
+    """
+    Save the model and tokenizer.
+    """
+    os.makedirs(save_dir, exist_ok=True)
+    model.save_pretrained(save_dir)
+    tokenizer.save_pretrained(save_dir)
+    print(f"Model and tokenizer saved at {save_dir}")