Add loading dataset

.gitignore

@@ -1,3 +1,4 @@
 env
 data/raw/tweet_eval_sentiment
-data/raw/youtube-comment-sentiment
+data/raw/youtube-comment-sentiment
+data/processed

src/app.py

@@ -7,6 +7,7 @@ from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from datasets import load_dataset, load_from_disk
 import torch
 import random
+import subprocess
 
 # Caricamento del modello e dei dati se già scaricati
 MODEL= "cardiffnlp/twitter-roberta-base-sentiment-latest"
@@ -20,23 +21,18 @@ model = AutoModelForSequenceClassification.from_pretrained(MODEL)
 labels = ["negative", "neutral", "positive"]
 
 
-if not os.path.exists(TWEET_PROCESSED_PATH):
-    tweet_eval = load_dataset("tweet_eval", "sentiment")
-    raise FileNotFoundError(
-        f"Dati non trovati in {TWEET_PROCESSED_PATH}. "
-        "Esegui src/data_preparation.py per crearlo."
-    ) 
 
+# TWEET EVAL
+if not os.path.exists(TWEET_PROCESSED_PATH):
+    print(f"Dataset Tweet Eval non trovato in {TWEET_PROCESSED_PATH}. Lo genero...")
+    subprocess.run(["python", "src/data_preparation.py", "tweet_eval"], check=True)
 tweet_eval = load_from_disk(TWEET_PROCESSED_PATH)
 
 
+# YOUTUBE COMMENTS
 if not os.path.exists(YT_PROCESSED_PATH):
-    youtube_ds = load_dataset("AmaanP314/youtube-comment-sentiment")
-    raise FileNotFoundError(
-        f"Dati non trovati in {YT_PROCESSED_PATH}. "
-        "Esegui src/data_preparation.py per crearlo."
-    ) 
-
+    print(f" Dataset YouTube non trovato in {YT_PROCESSED_PATH}. Lo genero...")
+    subprocess.run(["python", "src/data_preparation.py", "youtube"], check=True)
 youtube_ds = load_from_disk(YT_PROCESSED_PATH)
 
 app = FastAPI(

src/data_preparation.py

@@ -1,5 +1,6 @@
 from datasets import load_dataset
 from transformers import AutoTokenizer
+import argparse
 import re
 import os
 
@@ -33,19 +34,6 @@ def map_label(label):
     return label
 
 
-# Download tweet_eval
-tweet_eval = load_dataset("tweet_eval", "sentiment")
-# Download youtub comment dataset
-youtube = load_dataset("AmaanP314/youtube-comment-sentiment")
-
-
-tweet_eval = tweet_eval.map(lambda x: {"text": clean_text(x["text"])})
-youtube = youtube.map(lambda x: {"text": clean_text(x["CommentText"])})
-
-
-youtube = youtube.map(lambda x: {"label": map_label(x["Sentiment"])})
-
-
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 
 def tokenize_function(examples):
@@ -61,3 +49,32 @@ youtube_tokenized = youtube.map(tokenize_function, batched=True)
 
 tweet_tokenized.save_to_disk(os.path.join(PROCESSED_DIR, "tweet_eval_tokenized"))
 youtube_tokenized.save_to_disk(os.path.join(PROCESSED_DIR, "youtube_tokenized"))
+
+def prepare_tweet_eval(tokenizer, output_path):
+    print("Scarico e preparo il dataset Tweet Eval...")
+    ds = load_dataset("tweet_eval", "sentiment")
+    ds=ds.map(lambda x: {"text": clean_text(x["text"])})
+    ds=ds.map(tokenize_function, batched=True)
+    ds.save_to_disk(output_path)
+    print(f"Dataset Tweet Eval salvato in {output_path}")
+
+def prepare_youtube(tokenizer, output_path):
+    print("Scarico e preparo il dataset YouTube Comments...")
+    ds = load_dataset("AmaanP314/youtube-comment-sentiment")
+    ds = ds.map(lambda x: {"text": clean_text(x["CommentText"])})
+    ds = ds.map(lambda x: {"label": map_label(x["Sentiment"])})
+    ds.save_to_disk(output_path)
+    print(f"Dataset YouTube salvato in {output_path}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Prepara dataset per sentiment analysis.")
+    parser.add_argument("dataset", choices=["tweet_eval", "youtube"], help="Nome del dataset da preparare.")
+    args = parser.parse_args()
+
+    tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
+
+    if args.dataset == "tweet_eval":
+        prepare_tweet_eval(tokenizer, "data/processed/tweet_eval_tokenized")
+    elif args.dataset == "youtube":
+        prepare_youtube(tokenizer, "data/processed/youtube_comments")