Update app.py

app.py

@@ -1,138 +1,92 @@
 import os
-import logging
-import csv
-import shutil
 import nltk
-import pandas as pd
+import csv
+import logging
 from tqdm import tqdm
 import gradio as gr
-from datasets import Dataset
 from transformers import pipeline
-from huggingface_hub import HfApi
-
-# ---------------------- Logging Setup ----------------------
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s [%(levelname)s] %(message)s",
-    handlers=[logging.StreamHandler()]
-)
-
-# ---------------------- NLTK Setup ----------------------
-def download_nltk():
-    nltk.download("all")
-  #  nltk.download("punkt")
-    logging.info("NLTK resources downloaded.")
-
-download_nltk()
-
-# ---------------------- Data Preparation ----------------------
-def get_all_words():
-    from nltk.corpus import words as nltk_words
-    all_words = nltk_words.words()
-    logging.info(f"Got {len(all_words)} words from NLTK.")
-    return all_words
-
-def generate_meaning(word, generator):
-    prompt = f"Define the word '{word}' in one concise sentence."
-    try:
-        result = generator(prompt, max_length=50)[0]["generated_text"]
-        return result.strip()
-    except Exception as e:
-        logging.error(f"Error generating meaning for '{word}': {e}")
-        return ""
-
-def process_words(model_name, limit=None):
-    logging.info("Initializing Hugging Face text2text-generation pipeline...")
-    generator = pipeline("text2text-generation", model=model_name, device=-1)
-    words_list = get_all_words()
-    if limit:
-        words_list = words_list[:limit]
-    data = []
-    for word in tqdm(words_list, desc="Processing words"):
-        tokens = nltk.word_tokenize(word)
-        meaning = generate_meaning(word, generator)
-        data.append({
-            "tokenizer": tokens,
-            "words": word,
-            "meaning": meaning
-        })
-    logging.info("Finished processing words.")
-    return data
+from huggingface_hub import HfApi, upload_file
+
+# Setup Logging
+logging.basicConfig(filename='app.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Download NLTK Data
+nltk.download('punkt')
+
+# Constants
+HF_REPO = "katsukiai/DeepFocus-X3"
+TOKENIZER = 'bert-base-uncased'
+MODELS = ["bert-base-uncased", "gpt2", "roberta-base", "distilbert-base-uncased", "albert-base-v2"]  # Add more models as needed
+
+# Initialize Models
+models = {model: pipeline('feature-extraction', model=model) for model in MODELS}
+
+# Functions
+def process_text(text):
+    tokens = nltk.word_tokenize(text)
+    words = list(set(tokens))
+    means = {}
+    for word in tqdm(words, desc="Processing Words"):
+        word_means = {}
+        for model_name, model in models.items():
+            try:
+                output = model(word)
+                word_means[model_name] = output[0].mean().item()
+            except Exception as e:
+                logging.error(f"Error processing word {word} with model {model_name}: {e}")
+                word_means[model_name] = None
+        means[word] = word_means
+    return {"tokenizer": tokens, "words": words, "meaning": means}
 
 def save_to_csv(data, filename="output.csv"):
-    df = pd.DataFrame(data)
-    df.to_csv(filename, index=False)
-    logging.info(f"Saved CSV to {filename}.")
-    return filename
-
-# ---------------------- Push to Hugging Face ----------------------
-def push_dataset(csv_file, repo_id="katsukiai/DeepFocus-X3"):
-    repo_local_dir = "."
-    if not os.path.exists(repo_local_dir):
-        os.system(f"git clone https://huggingface.co/{repo_id} {repo_local_dir}")
-        logging.info("Repository cloned locally.")
-    shutil.copy(csv_file, os.path.join(repo_local_dir, csv_file))
-    current_dir = os.getcwd()
-    os.chdir(repo_local_dir)
-    os.system("git add .")
-    os.system('git commit -m "Update dataset"')
-    os.system("git push")
-    os.chdir(current_dir)
-    logging.info("Pushed dataset to Hugging Face repository.")
-
-def generate_all(model_name, word_limit):
-    try:
-        word_limit = int(word_limit)
-    except Exception:
-        word_limit = None
-    data = process_words(model_name, limit=word_limit)
-    csv_file = save_to_csv(data)
-    push_dataset(csv_file)
-    return csv_file
-
-# ---------------------- Gradio Interface Functions ----------------------
-def run_generate(model_name, word_limit):
-    output_csv = generate_all(model_name, word_limit)
-    return f"Generated and pushed CSV: {output_csv}"
-
-def about_tab_content():
-    about_text = (
-        "## DeepFocus-X3 Dataset Generator\n\n"
-        "This tool downloads all available words from the NLTK corpus, "
-        "generates concise meanings using a Hugging Face text-to-text generation model, "
-        "and converts the data into a CSV file. Finally, it pushes the CSV to the "
-        "[katsukiai/DeepFocus-X3](https://huggingface.co/datasets/katsukiai/DeepFocus-X3) repository."
-    )
-    return about_text
+    with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=["word", "tokenizer", "meanings"])
+        writer.writeheader()
+        for word in data['words']:
+            writer.writerow({
+                "word": word,
+                "tokenizer": data['tokenizer'],
+                "meanings": str(data['meaning'][word])
+            })
+
+def train_dataset():
+    text = "Your long text goes here..."
+    data = process_text(text)
+    save_to_csv(data)
+    logging.info("Dataset processed and saved to CSV.")
+
+def generate_report():
+    with open('app.log', 'r') as log_file:
+        log_content = log_file.read()
+    return log_content
+
+# Gradio Interface
+def generate_all(text):
+    data = process_text(text)
+    save_to_csv(data)
+    return f"Processed data saved to output.csv"
+
+iface = gr.Interface(
+    fn=[generate_all, generate_report],
+    inputs="text",
+    outputs=["text", "text"],
+    title="DeepFocus-X3",
+    tab_titles=["Generate All", "Logs"],
+    description="Generate processed data and view logs."
+)
 
-def settings_tab_content():
-    settings_text = (
-        "**Current Settings**\n\n"
-        "- Model: `google/flan-t5-xl`\n"
-        "- Word Limit: 50 (set to empty to process all words)\n"
-        "\nYou can update these settings in the Generate tab."
+# Run and Push to HuggingFace
+def run_and_push():
+    train_dataset()
+    api = HfApi()
+    api.create_repo(repo_id=HF_REPO, private=False, exist_ok=True)
+    upload_file(
+        path_or_fileobj="output.csv",
+        path_in_repo="output.csv",
+        repo_id=HF_REPO
     )
-    return settings_text
-
-# ---------------------- Gradio App ----------------------
-with gr.Blocks() as demo:
-    gr.Markdown("## DeepFocus-X3 Dataset Generator")
-
-    with gr.Tabs():
-        # About Tab
-        with gr.Tab("About"):
-            gr.Markdown(about_tab_content())
-
-        # Generate All Tab
-        with gr.Tab("Generate all"):
-            model_name_input = gr.Textbox(value="google/flan-t5-xl", label="Hugging Face Model Name for Means")
-            word_limit_input = gr.Textbox(value="50", label="Word Limit (Leave empty for all)")
-            generate_button = gr.Button("Generate and Push Dataset")
-            generate_output = gr.Textbox(label="Output")
-            generate_button.click(run_generate, inputs=[model_name_input, word_limit_input], outputs=generate_output)
-
-        # Settings Tab
-        with gr.Tab("Settings"):
-            gr.Markdown(settings_tab_content())
+    logging.info("Dataset pushed to HuggingFace.")
 
-demo.launch()
+if __name__ == "__main__":
+    iface.launch()
+    run_and_push()