Update app.py

app.py

@@ -1,42 +1,35 @@
 from huggingface_hub import InferenceClient
 import gradio as gr
-import PyPDF2
-import random
 import pandas as pd
-from io import BytesIO 
+import random
 import csv
 import os
-import io 
+import io
 import tempfile
 import re
 
 client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
 
-def extract_text_from_pdf(file):
-    pdf_reader = PyPDF2.PdfReader(file)
-    text = ""
-    for page in range(len(pdf_reader.pages)):
-        text += pdf_reader.pages[page].extract_text()
-    return text
+def extract_data_from_excel(file):
+    df = pd.read_excel(file)
+    return df.values.tolist()
 
 def save_to_csv(sentence, output, filename="synthetic_data.csv"):
     with open(filename, mode='a', newline='', encoding='utf-8') as file:
         writer = csv.writer(file)
         writer.writerow([sentence, output])
 
-
-
-def generate(file, temperature, max_new_tokens, top_p, repetition_penalty):
-    text = extract_text_from_pdf(file)
-    sentences = text.split('.')
+def generate(file, temperature, max_new_tokens, top_p, repetition_penalty, num_sentences=10000):
+    data = extract_data_from_excel(file)
+    sentences = [str(row) for row in data]  # Convert each row to a string
     random.shuffle(sentences)  # Shuffle sentences
 
     with tempfile.NamedTemporaryFile(mode='w', newline='', delete=False, suffix='.csv') as tmp:
         fieldnames = ['Original Sentence', 'Generated Sentence']
         writer = csv.DictWriter(tmp, fieldnames=fieldnames)
-        writer.writeheader()  
+        writer.writeheader()
 
-        for sentence in sentences:
+        for sentence in sentences[:num_sentences]:  # Process the first num_sentences sentences
             sentence = sentence.strip()
             if not sentence:
                 continue
@@ -68,16 +61,18 @@ def generate(file, temperature, max_new_tokens, top_p, repetition_penalty):
         tmp_path = tmp.name
 
     return tmp_path
-gr.Interface( 
+
+gr.Interface(
     fn=generate,
     inputs=[
-        gr.File(label="Upload PDF File", file_count="single", file_types=[".pdf"]),
+        gr.File(label="Upload Excel File", file_count="single", file_types=[".xlsx", ".xls"]),
         gr.Slider(label="Temperature", value=0.9, minimum=0.0, maximum=1.0, step=0.05, interactive=True, info="Higher values produce more diverse outputs"),
         gr.Slider(label="Max new tokens", value=256, minimum=0, maximum=5120, step=64, interactive=True, info="The maximum numbers of new tokens"),
         gr.Slider(label="Top-p (nucleus sampling)", value=0.95, minimum=0.0, maximum=1, step=0.05, interactive=True, info="Higher values sample more low-probability tokens"),
         gr.Slider(label="Repetition penalty", value=1.0, minimum=1.0, maximum=2.0, step=0.1, interactive=True, info="Penalize repeated tokens"),
+        gr.Slider(label="Number of sentences", value=10000, minimum=1, maximum=100000, step=1000, interactive=True, info="The number of sentences to generate"),
     ],
-    outputs=gr.File(label="Synthetic Data "),
+    outputs=gr.File(label="Synthetic Data"),
     title="SDG",
     description="AYE QABIL.",
     allow_flagging="never",