Update fine_tuning.py

fine_tuning.py

@@ -1,37 +1,11 @@
-import re
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
-from nltk.stem import PorterStemmer
 from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments
 from datasets import Dataset
 from sklearn.model_selection import train_test_split
-import nltk
 
-
-# Stop words ve stemmer
-stop_words = set(stopwords.words('english'))
-ps = PorterStemmer()
-
-# Metni temizleme fonksiyonu
-def clean_text_for_education(text):
-    text = re.sub(r'[^\w\s]', '', text)  # Noktalama işaretlerini temizler
-    text = re.sub(r'\d+', '', text)  # Sayıları temizler
-    text = text.lower()  # Küçük harfe çevirir
-    text = " ".join([word for word in text.split() if word not in stop_words])  # Stopwords kaldırır
-    return text
-
-# Prompts okuma
-def read_prompts(file_path):
-    input_texts = []
-    target_texts = []
-    with open(file_path, "r", encoding="utf-8") as file:
-        lines = file.readlines()
-        for line in lines:
-            if line.startswith("input:"):
-                input_texts.append(line.replace("input:", "").strip())
-            elif line.startswith("target:"):
-                target_texts.append(line.replace("target:", "").strip())
-    return input_texts, target_texts
+# Tokenizer ve model yükleme
+model_name = "t5-base"
+tokenizer = T5Tokenizer.from_pretrained(model_name)
+model = T5ForConditionalGeneration.from_pretrained(model_name)
 
 # Dataset hazırlama
 def prepare_data(input_texts, target_texts, tokenizer):
@@ -55,48 +29,41 @@ def paraphrase_with_model(text, model, tokenizer):
     )
     return tokenizer.decode(output_ids[0], skip_special_tokens=True)
 
-# Tokenizer ve model yükleme
-model_name = "t5-base"
-tokenizer = T5Tokenizer.from_pretrained(model_name)
-model = T5ForConditionalGeneration.from_pretrained(model_name)
-
-# Veriyi okuma ve temizleme
-input_texts, target_texts = read_prompts("prompts.txt")
-input_texts_cleaned = [clean_text_for_education(text) for text in input_texts]
-target_texts_cleaned = [clean_text_for_education(text) for text in target_texts]
+# Eğitim fonksiyonu
+def fine_tune_model(input_texts, target_texts):
+    # Eğitim ve doğrulama verisini ayırma
+    train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts, target_texts, test_size=0.1)
 
-# Eğitim ve doğrulama verisini ayırma
-train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts_cleaned, target_texts_cleaned, test_size=0.1)
+    # Augmentasyon ve dataset hazırlama
+    augmented_input_texts = input_texts + [paraphrase_with_model(text, model, tokenizer) for text in input_texts[:10]]
+    augmented_target_texts = target_texts + [paraphrase_with_model(text, model, tokenizer) for text in target_texts[:10]]
+    
+    train_dataset = Dataset.from_dict(prepare_data(augmented_input_texts, augmented_target_texts, tokenizer))
+    val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels, tokenizer))
 
-# Augmentasyon ve dataset hazırlama
-augmented_input_texts = input_texts_cleaned + [paraphrase_with_model(text, model, tokenizer) for text in input_texts_cleaned[:10]]
-augmented_target_texts = target_texts_cleaned + [paraphrase_with_model(text, model, tokenizer) for text in target_texts_cleaned[:10]]
-train_dataset = Dataset.from_dict(prepare_data(augmented_input_texts, augmented_target_texts, tokenizer))
-val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels, tokenizer))
-
-# Eğitim argümanları
-training_args = TrainingArguments(
-    output_dir="./results",
-    evaluation_strategy="steps",
-    learning_rate=5e-5,
-    per_device_train_batch_size=4,
-    num_train_epochs=3,
-    save_steps=500,
-    logging_dir="./logs",
-    logging_steps=10
-)
+    # Eğitim argümanları
+    training_args = TrainingArguments(
+        output_dir="./results",
+        evaluation_strategy="steps",
+        learning_rate=5e-5,
+        per_device_train_batch_size=4,
+        num_train_epochs=3,
+        save_steps=500,
+        logging_dir="./logs",
+        logging_steps=10
+    )
 
-# Trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=val_dataset
-)
+    # Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=val_dataset
+    )
 
-# Eğitim
-trainer.train()
+    # Eğitim
+    trainer.train()
 
-# Model kaydetme
-model.save_pretrained("./fine_tuned_model")
-tokenizer.save_pretrained("./fine_tuned_model")
+    # Model kaydetme
+    model.save_pretrained("./fine_tuned_model")
+    tokenizer.save_pretrained("./fine_tuned_model")