Update fine_tuning.py

fine_tuning.py

@@ -1,158 +1,101 @@
 from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments
 from datasets import Dataset
 from sklearn.model_selection import train_test_split
-import random
 import re
 from nltk.corpus import stopwords
 from nltk.tokenize import word_tokenize
 from nltk.stem import PorterStemmer
 
-def clean_pdf_text(pdf_text):
-    # Satır başlarındaki ve sonlarındaki boşlukları temizleyelim
-    pdf_text = pdf_text.strip()
-    
-    # Gereksiz satır aralıklarını kaldırma
-    pdf_text = re.sub(r'\n+', ' ', pdf_text)
-    
-    # Sayfa numarası gibi gereksiz kısımları kaldırma (örneğin 'Page 1', 'Page 2' gibi)
-    pdf_text = re.sub(r'\bPage \d+\b', '', pdf_text)
-    
-    return pdf_text
-
-def pdf_to_text(pdf_path):
-    """ Converts PDF to text """
-    pdf_text = extract_text(pdf_path)
-    return clean_pdf_text(pdf_text)
-
-# Stop words
+# Stop words ve stemmer
 stop_words = set(stopwords.words('english'))
-
-# Stemmer
 ps = PorterStemmer()
 
 # Metni temizleme fonksiyonu
 def clean_text(text):
-    # Noktalama işaretlerini kaldırma
     text = re.sub(r'[^\w\s]', '', text)
-    
-    # Sayıları kaldırma
     text = re.sub(r'\d+', '', text)
-    
-    # Küçük harfe çevirme
     text = text.lower()
-
-    # Stop words kaldırma
     text = " ".join([word for word in text.split() if word not in stop_words])
-    
-    # Kelimeleri köklerine indirgeme (stemming)
     text = " ".join([ps.stem(word) for word in word_tokenize(text)])
-    
     return text
 
+# Prompts okuma
+def read_prompts(file_path):
+    input_texts = []
+    target_texts = []
+    with open(file_path, "r", encoding="utf-8") as file:
+        lines = file.readlines()
+        for line in lines:
+            if line.startswith("input:"):
+                input_texts.append(line.replace("input:", "").strip())
+            elif line.startswith("target:"):
+                target_texts.append(line.replace("target:", "").strip())
+    return input_texts, target_texts
+
+# Dataset hazırlama
+def prepare_data(input_texts, target_texts):
+    inputs = tokenizer(input_texts, max_length=512, truncation=True, padding="max_length")
+    targets = tokenizer(target_texts, max_length=512, truncation=True, padding="max_length")
+    return {"input_ids": inputs["input_ids"], "attention_mask": inputs["attention_mask"], "labels": targets["input_ids"]}
+
 # Paraphrasing fonksiyonu
 def paraphrase_with_model(text, model, tokenizer):
-    prompt = "paraphrase: " + text  # T5 modeline paraphrasing görevi verdiğimizi belirtiyoruz.
-    
-    # Tokenizer ile metni tokenlara dönüştürme
+    prompt = "paraphrase: " + text
     inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
-    
-    # Sampling yöntemi ile model çalıştırma
     output_ids = model.generate(
         inputs["input_ids"],
-        do_sample=True,  # Sampling modunu aktif hale getirme
-        top_k=50,  # Top-k sampling
-        top_p=0.95,  # Top-p sampling
-        temperature=1.0,  # Daha fazla çeşitlilik için temperature
-        max_length=150,  # Maksimum cümle uzunluğu
-        no_repeat_ngram_size=2,  # Aynı n-gramların tekrarını engelle
-        early_stopping=True  # Daha erken durdurma
+        do_sample=True,
+        top_k=50,
+        top_p=0.95,
+        temperature=1.0,
+        max_length=150,
+        no_repeat_ngram_size=2,
+        early_stopping=True
     )
-    
-    # Modelin çıktısını decode ederek metni çözme, maksimum uzunluğu sınırlandırın
-    paraphrased_text = tokenizer.decode(output_ids[0], skip_special_tokens=True, max_length=150)
-
-    
-    return paraphrased_text
+    return tokenizer.decode(output_ids[0], skip_special_tokens=True, max_length=150)
 
-# Tokenizer ve modelin yüklenmesi
-model_name = "google-t5/t5-base"
+# Tokenizer ve model yükleme
+model_name = "t5-base"
 tokenizer = T5Tokenizer.from_pretrained(model_name)
 model = T5ForConditionalGeneration.from_pretrained(model_name)
 
-def prepare_data(input_texts, target_texts):
-    inputs = tokenizer(input_texts, max_length=512, truncation=True, padding=True, return_tensors="pt")
-    targets = tokenizer(target_texts, max_length=512, truncation=True, padding=True, return_tensors="pt")
-    
-    # Labels olarak hedef metinleri ayarlıyoruz
-    inputs["labels"] = targets["input_ids"]
-    
-    # Attention maskeleri de dahil et
-    inputs["attention_mask"] = inputs["attention_mask"]
-    
-    return inputs
-
-
-# Eğitim verileri
-input_texts = [ 
-    "Site Reliability Engineering is a concept born at Google. Ben Trainor's team of seven people started it in 2003 to keep Google.com running reliably.",
-    "Reliability is critical for any system. Without reliability, even the best features are useless as users can't access them.",
-    "SRE teams at Google handle large-scale systems with efficiency, working closely with developers to ensure scalability, reliability, and cost-effectiveness.",
-    "Site Reliability Engineering treats operations as a software engineering problem, making it distinct from traditional operations teams."]
-target_texts = [ 
-    "SRE was introduced at Google in 2003 by Ben Trainor's team to ensure the reliability of Google.com.",
-    "Reliability is essential for a system to be usable; without it, features lose value.",
-    "Google's SRE teams collaborate with developers to manage large-scale systems efficiently and reliably.",
-    "SRE approaches operations as a software engineering task, revolutionizing traditional operational methods."]
-
-# Veriyi temizleme
+# Veriyi okuma ve temizleme
+input_texts, target_texts = read_prompts("prompts.txt")
 input_texts_cleaned = [clean_text(text) for text in input_texts]
 target_texts_cleaned = [clean_text(text) for text in target_texts]
 
 # Eğitim ve doğrulama verisini ayırma
 train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts_cleaned, target_texts_cleaned, test_size=0.1)
 
-# Eğitim ve doğrulama verilerini hazırlama
-train_dataset = Dataset.from_dict(prepare_data(train_texts, train_labels))
+# Augmentasyon ve dataset hazırlama
+augmented_input_texts = input_texts_cleaned[:10] + [paraphrase_with_model(text, model, tokenizer) for text in input_texts_cleaned[:10]]
+augmented_target_texts = target_texts_cleaned[:10] + [paraphrase_with_model(text, model, tokenizer) for text in target_texts_cleaned[:10]]
+train_dataset = Dataset.from_dict(prepare_data(augmented_input_texts, augmented_target_texts))
 val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels))
 
-# Eğitim verisini augment etmek (opsiyonel)
-augmented_input_texts = [paraphrase_with_model(text, model, tokenizer) for text in input_texts_cleaned[:10]]  # Daha küçük bir örnekle başla
-augmented_target_texts = [paraphrase_with_model(text, model, tokenizer) for text in target_texts_cleaned[:10]]  # Aynı şekilde
-augmented_dataset = prepare_data(augmented_input_texts, augmented_target_texts)
-train_dataset = Dataset.from_dict(augmented_dataset)
-
 # Eğitim argümanları
 training_args = TrainingArguments(
     output_dir="./results",
     evaluation_strategy="steps",
     learning_rate=5e-5,
     per_device_train_batch_size=4,
-    per_device_eval_batch_size=4,
     num_train_epochs=3,
-    weight_decay=0.01,
-    save_steps=500,  # Modeli her 500 adımda bir kaydet
-    eval_steps=500,  # Değerlendirmeyi her 500 adımda bir yap
-    save_total_limit=2,
+    save_steps=500,
     logging_dir="./logs",
-    logging_steps=10,
-    load_best_model_at_end=True
+    logging_steps=10
 )
 
-# Trainer tanımı
+# Trainer
 trainer = Trainer(
     model=model,
     args=training_args,
     train_dataset=train_dataset,
-    eval_dataset=val_dataset  # Değerlendirme kümesi eklenmeli
+    eval_dataset=val_dataset
 )
 
-# Fine-tuning başlatma
+# Eğitim
 trainer.train()
 
-# Fine-tuned modelin kaydedilmesi
+# Model kaydetme
 model.save_pretrained("./fine_tuned_model")
-tokenizer.save_pretrained("./fine_tuned_model")
-
-# Fine-tuned modelinizi yükleyin
-model = T5ForConditionalGeneration.from_pretrained("./fine_tuned_model")
-tokenizer = T5Tokenizer.from_pretrained("./fine_tuned_model")
+tokenizer.save_pretrained("./fine_tuned_model")