Create fine_tune.py

fine_tune.py

@@ -0,0 +1,46 @@
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config
+from transformers import TextDataset, DataCollatorForLanguageModeling
+from transformers import Trainer, TrainingArguments
+
+# Load pre-trained GPT-2 model and tokenizer
+model_name = "gpt2"  # or "gpt2-medium", "gpt2-large", depending on your resources
+model = GPT2LMHeadModel.from_pretrained(model_name)
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+
+# Load your dataset
+train_dataset = TextDataset(
+    tokenizer=tokenizer,
+    file_path="path/to/your/dataset.txt",
+    block_size=128  # Adjust as needed
+)
+
+# Prepare data collator
+data_collator = DataCollatorForLanguageModeling(
+    tokenizer=tokenizer,
+    mlm=False
+)
+
+# Define training arguments
+training_args = TrainingArguments(
+    output_dir="./fine-tuned-gpt2",
+    overwrite_output_dir=True,
+    num_train_epochs=3,  # Adjust as needed
+    per_device_train_batch_size=4,  # Adjust based on GPU memory
+    save_steps=10_000,  # Save model checkpoints
+    save_total_limit=2,
+)
+
+# Initialize Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    data_collator=data_collator,
+    train_dataset=train_dataset,
+)
+
+# Fine-tune the model
+trainer.train()
+
+# Save the fine-tuned model
+model.save_pretrained("./fine-tuned-gpt2")
+tokenizer.save_pretrained("./fine-tuned-gpt2")