Update app.py

app.py

@@ -1,97 +1,27 @@
-import re
 import spacy
-from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments
-from datasets import Dataset
-from sklearn.model_selection import train_test_split
-from spacy.cli import download
+from transformers import T5Tokenizer
+from fine_tuning import fine_tune_model  # fine_tuning.py'deki fonksiyonu içe aktar
 
-# Spacy modelini indir ve yükle
-download("en_core_web_sm")
+# spaCy modelini yükle
 nlp = spacy.load("en_core_web_sm")
 
-# Metni temizleme fonksiyonu
-def clean_text_for_education_with_spacy(text):
+def clean_text_with_spacy(text):
     doc = nlp(text)
     tokens = [token.text for token in doc if not token.is_stop and not token.is_punct]
     return " ".join(tokens)
 
-# Prompts okuma
-def read_prompts(file_path):
-    input_texts = []
-    target_texts = []
-    with open(file_path, "r", encoding="utf-8") as file:
-        lines = file.readlines()
-        for line in lines:
-            if line.startswith("input:"):
-                input_texts.append(line.replace("input:", "").strip())
-            elif line.startswith("target:"):
-                target_texts.append(line.replace("target:", "").strip())
-    return input_texts, target_texts
-
-# Dataset hazırlama
-def prepare_data(input_texts, target_texts, tokenizer):
-    inputs = tokenizer(input_texts, max_length=256, truncation=True, padding="max_length")
-    targets = tokenizer(target_texts, max_length=256, truncation=True, padding="max_length")
-    return {"input_ids": inputs["input_ids"], "attention_mask": inputs["attention_mask"], "labels": targets["input_ids"]}
-
-# Paraphrasing fonksiyonu
-def paraphrase_with_model(text, model, tokenizer):
-    prompt = "Teach the following content: " + text
-    inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
-    output_ids = model.generate(
-        inputs["input_ids"],
-        do_sample=True,
-        top_k=50,
-        top_p=0.95,
-        temperature=1.0,
-        max_length=150,
-        no_repeat_ngram_size=2,
-        early_stopping=True
-    )
-    return tokenizer.decode(output_ids[0], skip_special_tokens=True)
-
-# Tokenizer ve model yükleme
-model_name = "t5-base"
-tokenizer = T5Tokenizer.from_pretrained(model_name)
-model = T5ForConditionalGeneration.from_pretrained(model_name)
-
-# Veriyi okuma ve temizleme
-input_texts, target_texts = read_prompts("prompts.txt")
-input_texts_cleaned = [clean_text_for_education_with_spacy(text) for text in input_texts]
-target_texts_cleaned = [clean_text_for_education_with_spacy(text) for text in target_texts]
-
-# Eğitim ve doğrulama verisini ayırma
-train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts_cleaned, target_texts_cleaned, test_size=0.1)
-
-# Augmentasyon ve dataset hazırlama
-augmented_input_texts = input_texts_cleaned + [paraphrase_with_model(text, model, tokenizer) for text in input_texts_cleaned[:10]]
-augmented_target_texts = target_texts_cleaned + [paraphrase_with_model(text, model, tokenizer) for text in target_texts_cleaned[:10]]
-train_dataset = Dataset.from_dict(prepare_data(augmented_input_texts, augmented_target_texts, tokenizer))
-val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels, tokenizer))
-
-# Eğitim argümanları
-training_args = TrainingArguments(
-    output_dir="./results",
-    evaluation_strategy="steps",
-    learning_rate=5e-5,
-    per_device_train_batch_size=4,
-    num_train_epochs=3,
-    save_steps=500,
-    logging_dir="./logs",
-    logging_steps=10
-)
-
-# Trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=val_dataset
-)
-
-# Eğitim
-trainer.train()
-
-# Model kaydetme
-model.save_pretrained("./fine_tuned_model")
-tokenizer.save_pretrained("./fine_tuned_model")
+# Temizlenmiş metni modelinize göndermek için fonksiyon
+def process_input_for_fine_tuning(input_texts, target_texts):
+    # Metni temizle
+    cleaned_input_texts = [clean_text_with_spacy(text) for text in input_texts]
+    cleaned_target_texts = [clean_text_with_spacy(text) for text in target_texts]
+    
+    # Temizlenmiş metni fine-tuning için gönder
+    fine_tune_model(cleaned_input_texts, cleaned_target_texts)
+
+# Örnek metinler
+input_texts = ["This is a sample input text.", "Another input text here."]
+target_texts = ["This is the target output.", "Target output for second example."]
+
+# Temizlenmiş veriyi fine_tuning.py'ye göndermek için işlemi başlat
+process_input_for_fine_tuning(input_texts, target_texts)