Update app.py

app.py

@@ -3,14 +3,14 @@ import csv
 import json
 import logging
 import gradio as gr
+import pandas as pd
 from tqdm import tqdm
 import nltk
 from nltk.tokenize import word_tokenize
 from nltk.corpus import wordnet
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-from huggingface_hub import HfApi, Repository, login
+from huggingface_hub import HfApi, login
 from datasets import Dataset
-import pandas as pd
 from datetime import datetime
 import secrets
 
@@ -31,52 +31,43 @@ error_log_file = os.path.join(error_dir, f"errors_{datetime.now().strftime('%Y%m
 def log_error(error_msg):
     with open(error_log_file, 'a') as f:
         f.write(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - ERROR - {error_msg}\n")
-    try:
-        api = HfApi()
-        api.upload_file(
-            path_or_fileobj=error_log_file,
-            path_in_repo=f"errors_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log",
-            repo_id="katsukiai/errors",
-            repo_type="dataset"
-        )
-    except Exception as e:
-        logging.error(f"Failed to upload error log: {str(e)}")
-
-tokenizer = AutoTokenizer.from_pretrained("amd/Instella-3B-Instruct",trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("amd/Instella-3B-Instruct",trust_remote_code=True)
-meaning_generator = pipeline("text2text-generation", model="google/flan-t5-large")
 
 HF_TOKEN = os.getenv("HF_TOKEN", secrets.token_hex(16))
 login(token=HF_TOKEN)
 
+tokenizer = AutoTokenizer.from_pretrained("amd/Instella-3B-Instruct", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("amd/Instella-3B-Instruct", trust_remote_code=True)
+meaning_generator = pipeline("text2text-generation", model="google/flan-t5-large")
+
 dataset_dir = "dataset"
 os.makedirs(dataset_dir, exist_ok=True)
 csv_file = os.path.join(dataset_dir, "deepfocus_data.csv")
+parquet_file = os.path.join(dataset_dir, "deepfocus_data.parquet")
 
 def process_text_to_csv(input_text):
     try:
         tokens = word_tokenize(input_text.lower())
         words = list(set(tokens))
         data = []
+
+        existing_df = pd.read_parquet(parquet_file) if os.path.exists(parquet_file) else pd.DataFrame(columns=["words", "meaning"])
+        existing_words = set(existing_df["words"].tolist())
+
         for word in tqdm(words, desc="Processing words"):
-            meanings = []
-            synsets = wordnet.synsets(word)
-            if synsets:
-                meanings = [syn.definition() for syn in synsets[:3]]
-            else:
-                try:
-                    generated_meaning = meaning_generator(f"Define the word '{word}'", max_length=100)[0]['generated_text']
-                    meanings.append(generated_meaning)
-                except Exception as e:
-                    log_error(f"Meaning generation failed for '{word}': {str(e)}")
-            data.append({"tokenizer": tokens, "words": word, "meaning": meanings})
-        
-        with open(csv_file, 'w', newline='', encoding='utf-8') as f:
-            writer = csv.DictWriter(f, fieldnames=["tokenizer", "words", "meaning"])
-            writer.writeheader()
-            writer.writerows(data)
-        
-        logging.info(f"Dataset saved to {csv_file}")
+            if word in existing_words:
+                continue
+
+            meanings = [syn.definition() for syn in wordnet.synsets(word)[:3]] or \
+                       [meaning_generator(f"Define the word '{word}'", max_length=100)[0]['generated_text']]
+
+            data.append({"words": word, "meaning": meanings})
+
+        if data:
+            new_df = pd.DataFrame(data)
+            combined_df = pd.concat([existing_df, new_df], ignore_index=True)
+            combined_df.to_parquet(parquet_file, index=False)
+            combined_df.to_csv(csv_file, index=False, encoding='utf-8')
+
         return data
     except Exception as e:
         log_error(f"Error in process_text_to_csv: {str(e)}")
@@ -84,9 +75,8 @@ def process_text_to_csv(input_text):
 
 def upload_to_huggingface():
     try:
-        dataset = Dataset.from_csv(csv_file)
+        dataset = Dataset.from_parquet(parquet_file)
         dataset.push_to_hub("katsukiai/DeepFocus-X3", token=HF_TOKEN)
-        logging.info("Dataset uploaded to Hugging Face")
     except Exception as e:
         log_error(f"Error uploading to Hugging Face: {str(e)}")
         raise
@@ -103,10 +93,7 @@ def generate_output(input_text):
 def view_logs():
     try:
         log_files = os.listdir(log_dir)
-        log_content = ""
-        for log_file in log_files:
-            with open(os.path.join(log_dir, log_file), 'r') as f:
-                log_content += f"\n\n--- {log_file} ---\n\n{f.read()}"
+        log_content = "".join(f"\n\n--- {log_file} ---\n\n{open(os.path.join(log_dir, log_file), 'r').read()}" for log_file in log_files)
         return log_content
     except Exception as e:
         log_error(f"Error in view_logs: {str(e)}")
@@ -117,13 +104,7 @@ with gr.Blocks(title="DeepFocus-X3") as demo:
     
     with gr.Tabs():
         with gr.TabItem("About"):
-            gr.Markdown("""
-            ## About DeepFocus-X3
-            This application processes text, tokenizes it, extracts unique words, generates meanings, and uploads the dataset to Hugging Face.
-            - Uses NLTK for tokenization and WordNet for meanings.
-            - Leverages DeepSeek AI for long text processing and Google FLAN-T5 for meaning generation.
-            - Logs all activities and errors, with error logs uploaded to Hugging Face.
-            """)
+            gr.Markdown("## About DeepFocus-X3\nThis application processes text, tokenizes it, extracts unique words, generates meanings, and uploads the dataset to Hugging Face.")
         
         with gr.TabItem("Generate all"):
             input_text = gr.Textbox(label="Input Text", lines=10)
@@ -137,4 +118,4 @@ with gr.Blocks(title="DeepFocus-X3") as demo:
             view_logs_btn = gr.Button("View Logs")
             view_logs_btn.click(fn=view_logs, inputs=None, outputs=log_output)
 
-demo.launch()
+demo.launch()