Update app.py

app.py

@@ -4,6 +4,8 @@ import PyPDF2
 import random
 import pandas as pd
 from io import StringIO
+import csv
+import os
 
 # Initialize the inference client with your chosen model
 client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
@@ -15,14 +17,25 @@ def extract_text_from_pdf(file):
         text += pdf_reader.pages[page].extract_text()
     return text
 
-def generate_synthetic_data(sentences, temperature, max_new_tokens, top_p, repetition_penalty):
-    synthetic_data = []
+def save_to_csv(sentence, output, filename="synthetic_data.csv"):
+    with open(filename, mode='a', newline='', encoding='utf-8') as file:
+        writer = csv.writer(file)
+        writer.writerow([sentence, output])
+
+def generate(file, temperature, max_new_tokens, top_p, repetition_penalty):
+    text = extract_text_from_pdf(file)
+    sentences = text.split('.')
+    random.shuffle(sentences)  # Shuffle sentences
+
+    # CSV dosyası için başlık
+    if not os.path.exists("synthetic_data.csv"):
+        save_to_csv("Original Sentence", "Synthetic Data")
+
     for sentence in sentences:
-        # Trim whitespace and skip if the sentence is empty
         sentence = sentence.strip()
         if not sentence:
             continue
-        
+
         generate_kwargs = {
             "temperature": temperature,
             "max_new_tokens": max_new_tokens,
@@ -37,26 +50,12 @@ def generate_synthetic_data(sentences, temperature, max_new_tokens, top_p, repet
             output = ""
             for response in stream:
                 output += response.token.text
-            synthetic_data.append(output)
+            save_to_csv(sentence, output)
         except Exception as e:
             print(f"Error generating data for sentence '{sentence}': {e}")
-            # Optionally, append a placeholder or error message to `synthetic_data` to maintain alignment with input sentences
-            synthetic_data.append(f"Error: {e}")
-    return synthetic_data
-
-def generate(file, temperature, max_new_tokens, top_p, repetition_penalty):
-    # Extract text and split into sentences
-    text = extract_text_from_pdf(file)
-    sentences = text.split('.')
-    random.shuffle(sentences)  # Shuffle sentences
-
-    synthetic_data = generate_synthetic_data(sentences, temperature, max_new_tokens, top_p, repetition_penalty)
+            save_to_csv(sentence, f"Error: {e}")
 
-    # Convert synthetic data to CSV
-    df = pd.DataFrame(synthetic_data, columns=["Synthetic Data"])
-    csv_buffer = StringIO()
-    df.to_csv(csv_buffer, index=False)
-    return gr.File(value=csv_buffer.getvalue(), file_name="synthetic_data.csv")
+    return gr.File(value="synthetic_data.csv", file_name="synthetic_data.csv")
 
 gr.Interface(
     fn=generate,
@@ -69,6 +68,6 @@ gr.Interface(
     ],
     outputs="file",
     title="Synthetic Data Generation",
-    description="This tool generates synthetic data from the sentences in your PDF.",
+    description="This tool generates synthetic data from the sentences in your PDF and saves it to a CSV file.",
     allow_flagging="never",
 ).launch()