Add YouTube Comment Analyzer with FastAPI

Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies for PyTorch and transformers
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first for better caching
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create cache directory for models
+RUN mkdir -p /app/cache /app/data/cache
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV HF_HOME=/app/cache
+ENV TRANSFORMERS_CACHE=/app/cache
+ENV TORCH_HOME=/app/cache
+ENV PYTHONPATH=/app
+
+# Expose Hugging Face Spaces default port
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+  CMD curl -f http://localhost:7860/health || exit 1
+
+# Start the application
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

data/fetch_comments.py

@@ -0,0 +1,271 @@
+import requests
+import re
+import time
+import sqlite3
+import os
+from datetime import datetime
+
+# Your YouTube API key
+API_KEY = "AIzaSyBwwLnHXKmL5VQhkKzumCS5r1Cbi3HdUro"
+
+def extract_video_id(url):
+    """Extract video ID from YouTube URL"""
+    patterns = [
+        r'(?:youtube\.com\/watch\?v=)([\w-]+)',
+        r'(?:youtu\.be\/)([\w-]+)',
+        r'(?:youtube\.com\/embed\/)([\w-]+)',
+        r'(?:youtube\.com\/v\/)([\w-]+)',
+        r'(?:youtube\.com\/watch\?.*v=)([\w-]+)'
+    ]
+    
+    for pattern in patterns:
+        match = re.search(pattern, url)
+        if match:
+            return match.group(1)
+    return None
+
+# Database setup for caching large comment batches
+DB_PATH = "youtube_cache.db"
+
+def init_cache_db():
+    """Initialize SQLite cache database for storing fetched comments"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS cached_comments (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            video_id TEXT,
+            comment_text TEXT,
+            fetched_at TIMESTAMP,
+            processed INTEGER DEFAULT 0,
+            sentiment TEXT,
+            sarcasm TEXT,
+            emotion TEXT
+        )
+    ''')
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_video_id ON cached_comments(video_id)
+    ''')
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_processed ON cached_comments(processed)
+    ''')
+    conn.commit()
+    conn.close()
+
+def save_comments_to_cache(video_id, comments):
+    """Save fetched comments to local cache"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    # Clear old cache for this video first (optional - for fresh analysis)
+    # cursor.execute("DELETE FROM cached_comments WHERE video_id = ?", (video_id,))
+    
+    for comment in comments:
+        cursor.execute('''
+            INSERT INTO cached_comments (video_id, comment_text, fetched_at, processed)
+            VALUES (?, ?, ?, 0)
+        ''', (video_id, comment, datetime.now()))
+    
+    conn.commit()
+    conn.close()
+    print(f"💾 Saved {len(comments)} comments to cache")
+
+def load_comments_from_cache(video_id, limit=None):
+    """Load comments from local cache"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    if limit:
+        cursor.execute('''
+            SELECT comment_text FROM cached_comments 
+            WHERE video_id = ? AND processed = 0
+            LIMIT ?
+        ''', (video_id, limit))
+    else:
+        cursor.execute('''
+            SELECT comment_text FROM cached_comments 
+            WHERE video_id = ? AND processed = 0
+        ''', (video_id,))
+    
+    comments = [row[0] for row in cursor.fetchall()]
+    conn.close()
+    return comments
+
+def get_cached_comment_count(video_id):
+    """Get count of cached comments for a video"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('''
+        SELECT COUNT(*) FROM cached_comments 
+        WHERE video_id = ? AND processed = 0
+    ''', (video_id,))
+    count = cursor.fetchone()[0]
+    conn.close()
+    return count
+
+def get_comments_from_url(video_url, max_results=500):
+    """
+    Fetch YouTube comments with pagination support
+    Can fetch up to 10000+ comments with caching
+    """
+    try:
+        video_id = extract_video_id(video_url)
+        
+        if not video_id:
+            print(f"Could not extract video ID from URL: {video_url}")
+            return []
+        
+        print(f"Video ID: {video_id}")
+        print(f"Requested comments: {max_results}")
+        
+        # Check cache first
+        init_cache_db()
+        cached_count = get_cached_comment_count(video_id)
+        
+        if cached_count >= max_results:
+            print(f"✅ Using {cached_count} cached comments (no API call needed)")
+            return load_comments_from_cache(video_id, max_results)
+        
+        print(f"📡 Fetching from YouTube API (cached: {cached_count}, need: {max_results})")
+        
+        all_comments = []
+        next_page_token = None
+        comments_fetched = 0
+        
+        # YouTube API max per request is 100
+        per_page = min(100, max_results)
+        page = 1
+        
+        while comments_fetched < max_results:
+            # Build URL with pagination
+            url = f"https://www.googleapis.com/youtube/v3/commentThreads?part=snippet&videoId={video_id}&maxResults={per_page}&key={API_KEY}"
+            if next_page_token:
+                url += f"&pageToken={next_page_token}"
+            
+            print(f"Fetching page {page}...", end=" ")
+            response = requests.get(url, timeout=30)
+            data = response.json()
+            
+            if "error" in data:
+                error_msg = data['error'].get('message', 'Unknown error')
+                print(f"\n❌ API Error: {error_msg}")
+                
+                if "quotaExceeded" in error_msg:
+                    print("⚠️ API quota exceeded. Using cached comments if available.")
+                    if all_comments:
+                        save_comments_to_cache(video_id, all_comments)
+                    return load_comments_from_cache(video_id, max_results)
+                elif "commentsDisabled" in error_msg:
+                    print("Comments are disabled for this video.")
+                break
+            
+            if "items" not in data or not data["items"]:
+                print("No more comments found")
+                break
+            
+            # Extract comments from this batch
+            for item in data["items"]:
+                comment = item["snippet"]["topLevelComment"]["snippet"]["textDisplay"]
+                # Clean HTML entities
+                comment = re.sub(r'<.*?>', '', comment)
+                comment = comment.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')
+                comment = comment.replace('&#39;', "'").replace('&quot;', '"')
+                all_comments.append(comment)
+                comments_fetched += 1
+                
+                if comments_fetched >= max_results:
+                    break
+            
+            print(f"✅ Got {len(all_comments)} comments so far")
+            
+            # Check if there are more pages
+            next_page_token = data.get("nextPageToken")
+            if not next_page_token:
+                print("No more pages available")
+                break
+            
+            page += 1
+            
+            # Small delay to avoid rate limiting
+            time.sleep(0.2)
+        
+        # Save to cache for future use
+        if all_comments:
+            save_comments_to_cache(video_id, all_comments)
+        
+        print(f"✅ Successfully fetched {len(all_comments)} comments from YouTube!")
+        return all_comments
+        
+    except requests.exceptions.Timeout:
+        print("Request timeout. Try again with fewer comments.")
+        return []
+    except Exception as e:
+        print(f"Error fetching comments: {e}")
+        return []
+
+# New function for lakhs of comments with resume support
+def get_comments_batch(video_url, batch_size=500, offset=0):
+    """
+    Fetch comments in batches for lakhs of comments
+    Returns (comments, has_more, total_fetched)
+    """
+    try:
+        video_id = extract_video_id(video_url)
+        if not video_id:
+            return [], False, 0
+        
+        init_cache_db()
+        
+        # Try to get from cache first
+        cached = load_comments_from_cache(video_id, batch_size)
+        if len(cached) >= batch_size:
+            return cached[:batch_size], True, len(cached)
+        
+        # Need to fetch more
+        all_comments = []
+        next_page_token = None
+        fetched = 0
+        
+        url = f"https://www.googleapis.com/youtube/v3/commentThreads?part=snippet&videoId={video_id}&maxResults=100&key={API_KEY}"
+        
+        # Skip to offset by paginating
+        pages_to_skip = offset // 100
+        for _ in range(pages_to_skip):
+            response = requests.get(url, timeout=30)
+            data = response.json()
+            next_page_token = data.get("nextPageToken")
+            if not next_page_token:
+                break
+            url = f"https://www.googleapis.com/youtube/v3/commentThreads?part=snippet&videoId={video_id}&maxResults=100&pageToken={next_page_token}&key={API_KEY}"
+        
+        while fetched < batch_size:
+            response = requests.get(url, timeout=30)
+            data = response.json()
+            
+            if "error" in data or "items" not in data:
+                break
+            
+            for item in data["items"]:
+                comment = item["snippet"]["topLevelComment"]["snippet"]["textDisplay"]
+                comment = re.sub(r'<.*?>', '', comment)
+                all_comments.append(comment)
+                fetched += 1
+                if fetched >= batch_size:
+                    break
+            
+            next_page_token = data.get("nextPageToken")
+            if not next_page_token:
+                break
+            
+            url = f"https://www.googleapis.com/youtube/v3/commentThreads?part=snippet&videoId={video_id}&maxResults=100&pageToken={next_page_token}&key={API_KEY}"
+            time.sleep(0.2)
+        
+        if all_comments:
+            save_comments_to_cache(video_id, all_comments)
+        
+        has_more = next_page_token is not None
+        return all_comments, has_more, len(all_comments)
+        
+    except Exception as e:
+        print(f"Error in batch fetch: {e}")
+        return [], False, 0

main.py

@@ -0,0 +1,298 @@
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List, Optional
+from data.fetch_comments import get_comments_from_url, get_comments_batch, init_cache_db
+from model.model_loader import (
+    predict_sentiment,
+    detect_sarcasm,
+    detect_emotion,
+    generate_summary,
+    extract_keywords,
+    process_comments_in_batches,
+    get_batch_stats
+)
+import time
+from datetime import datetime
+import asyncio
+from threading import Thread
+import os 
+
+app = FastAPI(title="YouTube Comment Analyzer API", version="2.0.0")
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Initialize cache DB on startup
+@app.on_event("startup")
+async def startup_event():
+    init_cache_db()
+    print("✅ Cache database initialized")
+
+class SingleCommentRequest(BaseModel):
+    text: str
+
+# Store background job status
+analysis_jobs = {}
+
+@app.get("/")
+async def root():
+    return {
+        "message": "YouTube Comment Analyzer API",
+        "status": "running",
+        "version": "2.0.0",
+        "max_comments": "UNLIMITED (supports lakhs of comments with caching)",
+        "endpoints": {
+            "/analyze_youtube": "GET - Analyze YouTube video comments (up to 1000 quickly)",
+            "/analyze_large": "POST - Analyze lakhs of comments asynchronously",
+            "/job_status/{job_id}": "GET - Check background job status",
+            "/predict": "POST - Analyze single comment",
+            "/health": "GET - Check API health"
+        }
+    }
+
+@app.get("/health")
+async def health_check():
+    return {
+        "status": "healthy",
+        "timestamp": datetime.now().isoformat(),
+        "models_loaded": True,
+        "cache_initialized": True
+    }
+
+@app.get("/analyze_youtube")
+async def analyze_youtube(url: str, limit: int = 500):
+    """
+    Analyze YouTube video comments
+    NOW SUPPORTS UP TO 10000+ COMMENTS (removed 1000 cap)
+    """
+    try:
+        # REMOVED THE 1000 CAP - Now supports more!
+        # if limit > 1000:
+        #     limit = 1000
+        
+        # Cap at 10000 for reasonable response time, but cache handles more
+        if limit > 10000:
+            print(f"⚠️ Large request: {limit} comments. This may take several minutes.")
+        
+        print(f"\n{'='*60}")
+        print(f"Analyzing YouTube URL: {url}")
+        print(f"Comment limit: {limit}")
+        print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+        print(f"{'='*60}")
+        
+        start_time = time.time()
+        
+        # Fetch real comments from YouTube (now with caching)
+        comments = get_comments_from_url(url, limit)
+        
+        if not comments:
+            return {
+                "total_comments": 0,
+                "summary": "No comments found for this video.",
+                "keywords": [],
+                "stats": {"positive": 0, "neutral": 0, "negative": 0},
+                "sentiment_score": 0,
+                "results": [],
+                "timestamp": datetime.now().isoformat(),
+                "processing_time": round(time.time() - start_time, 2)
+            }
+        
+        print(f"Processing {len(comments)} comments with ML models...")
+        
+        # Process in batches for better performance
+        all_results = []
+        stats = {"positive": 0, "neutral": 0, "negative": 0}
+        
+        # Use batch processing
+        for batch_results in process_comments_in_batches(comments, batch_size=100):
+            for result in batch_results:
+                all_results.append(result)
+                if result["sentiment"] == "POSITIVE":
+                    stats["positive"] += 1
+                elif result["sentiment"] == "NEGATIVE":
+                    stats["negative"] += 1
+                else:
+                    stats["neutral"] += 1
+        
+        # Generate summary and keywords
+        print("Generating AI summary...")
+        summary = generate_summary(comments[:200])  # Use more comments for better summary
+        
+        print("Extracting keywords...")
+        keywords = extract_keywords(comments[:300])  # Use more comments for better keywords
+        
+        # Calculate sentiment score
+        total = stats["positive"] + stats["neutral"] + stats["negative"]
+        sentiment_score = ((stats["positive"] - stats["negative"]) / total * 100) if total > 0 else 0
+        
+        processing_time = round(time.time() - start_time, 2)
+        
+        response_data = {
+            "total_comments": len(all_results),
+            "summary": summary,
+            "keywords": keywords[:20],  # More keywords
+            "stats": stats,
+            "sentiment_score": round(sentiment_score, 1),
+            "results": all_results,
+            "timestamp": datetime.now().isoformat(),
+            "processing_time": processing_time
+        }
+        
+        print(f"\n✅ Analysis Complete!")
+        print(f"   Total Comments: {len(all_results)}")
+        print(f"   Positive: {stats['positive']} ({stats['positive']/total*100:.1f}%)")
+        print(f"   Neutral: {stats['neutral']} ({stats['neutral']/total*100:.1f}%)")
+        print(f"   Negative: {stats['negative']} ({stats['negative']/total*100:.1f}%)")
+        print(f"   Sentiment Score: {sentiment_score:.1f}%")
+        print(f"   Processing Time: {processing_time} seconds")
+        print(f"{'='*60}\n")
+        
+        return response_data
+        
+    except Exception as e:
+        print(f"Error in analyze_youtube: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/analyze_large")
+async def analyze_large_scale(
+    url: str, 
+    max_comments: int = 100000,  # Now supports lakhs!
+    background_tasks: BackgroundTasks = None
+):
+    """
+    Analyze up to 100,000+ comments asynchronously
+    Returns job_id to check status
+    """
+    import uuid
+    job_id = str(uuid.uuid4())
+    
+    analysis_jobs[job_id] = {
+        "status": "pending",
+        "progress": 0,
+        "total": max_comments,
+        "started_at": datetime.now().isoformat()
+    }
+    
+    def run_large_analysis():
+        try:
+            analysis_jobs[job_id]["status"] = "running"
+            
+            # Fetch in batches
+            all_comments = []
+            offset = 0
+            batch_size = 1000
+            
+            while len(all_comments) < max_comments:
+                batch, has_more, fetched = get_comments_batch(url, batch_size, offset)
+                if not batch:
+                    break
+                all_comments.extend(batch)
+                offset += fetched
+                analysis_jobs[job_id]["progress"] = len(all_comments)
+                
+                if not has_more:
+                    break
+            
+            # Process in batches
+            all_results = []
+            stats = {"positive": 0, "neutral": 0, "negative": 0}
+            
+            for batch_results in process_comments_in_batches(all_comments, batch_size=200):
+                for result in batch_results:
+                    all_results.append(result)
+                    if result["sentiment"] == "POSITIVE":
+                        stats["positive"] += 1
+                    elif result["sentiment"] == "NEGATIVE":
+                        stats["negative"] += 1
+                    else:
+                        stats["neutral"] += 1
+                
+                analysis_jobs[job_id]["progress"] = len(all_results)
+            
+            analysis_jobs[job_id]["results"] = {
+                "total_comments": len(all_results),
+                "stats": stats,
+                "sentiment_score": ((stats["positive"] - stats["negative"]) / len(all_results) * 100) if len(all_results) > 0 else 0,
+                "summary": generate_summary(all_comments[:200]),
+                "keywords": extract_keywords(all_comments[:300])
+            }
+            analysis_jobs[job_id]["status"] = "completed"
+            analysis_jobs[job_id]["completed_at"] = datetime.now().isoformat()
+            
+        except Exception as e:
+            analysis_jobs[job_id]["status"] = "failed"
+            analysis_jobs[job_id]["error"] = str(e)
+    
+    # Run in background
+    thread = Thread(target=run_large_analysis)
+    thread.start()
+    
+    return {
+        "job_id": job_id,
+        "status": "started",
+        "message": f"Analysis started for up to {max_comments} comments. Use /job_status/{job_id} to check progress"
+    }
+
+@app.get("/job_status/{job_id}")
+async def get_job_status(job_id: str):
+    """Check status of large analysis job"""
+    if job_id not in analysis_jobs:
+        raise HTTPException(status_code=404, detail="Job not found")
+    
+    job = analysis_jobs[job_id]
+    response = {
+        "job_id": job_id,
+        "status": job["status"],
+        "progress": job.get("progress", 0),
+        "total": job.get("total", 0)
+    }
+    
+    if job["status"] == "completed":
+        response["results"] = job.get("results")
+        response["completed_at"] = job.get("completed_at")
+    elif job["status"] == "failed":
+        response["error"] = job.get("error")
+    
+    return response
+
+@app.post("/predict")
+async def predict_sentiment_endpoint(request: SingleCommentRequest):
+    """Analyze sentiment of a single comment"""
+    try:
+        sentiment = predict_sentiment(request.text)
+        sarcasm = detect_sarcasm(request.text)
+        emotion = detect_emotion(request.text)
+        
+        return {
+            "text": request.text,
+            "sentiment": sentiment,
+            "sarcasm": sarcasm,
+            "emotion": emotion,
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    
+if __name__ == "__main__":
+    import uvicorn
+    # Hugging Face Spaces uses port 7860
+    port = int(os.environ.get("PORT", 7860))
+    print("\n" + "="*60)
+    print("🚀 YouTube Comment Analyzer API v2.0 - Hugging Face Edition")
+    print("="*60)
+    print(f"Server starting on http://0.0.0.0:{port}")
+    print("✅ Supports 100,000+ comments with caching")
+    print("✅ SQLite cache for faster subsequent analysis")
+    print("="*60 + "\n")
+    uvicorn.run(app, host="0.0.0.0", port=port)
+    
+    

model/model_loader.py

@@ -0,0 +1,316 @@
+from transformers import pipeline
+from keybert import KeyBERT
+import torch
+import re
+from collections import Counter
+import numpy as np
+
+# Check if GPU is available
+device = 0 if torch.cuda.is_available() else -1
+print(f"Using device: {'GPU' if torch.cuda.is_available() else 'CPU'}")
+
+# Load sentiment analysis model
+print("Loading sentiment analysis model...")
+sentiment_model = pipeline(
+    "sentiment-analysis",
+    model="cardiffnlp/twitter-roberta-base-sentiment-latest",
+    device=device,
+    truncation=True,
+    max_length=512
+)
+
+# Load sarcasm detection model
+print("Loading sarcasm detection model...")
+sarcasm_model = pipeline(
+    "text-classification",
+    model="cardiffnlp/twitter-roberta-base-irony",
+    device=device,
+    truncation=True,
+    max_length=512
+)
+
+# Load emotion detection model
+print("Loading emotion detection model...")
+emotion_model = pipeline(
+    "text-classification",
+    model="j-hartmann/emotion-english-distilroberta-base",
+    device=device,
+    truncation=True,
+    max_length=512
+)
+
+# Load summarizer
+print("Loading summarizer model...")
+summarizer = pipeline(
+    "summarization",
+    model="facebook/bart-large-cnn",
+    device=device,
+    truncation=True,
+    max_length=1024
+)
+
+# Load keyword extractor
+print("Loading keyword extractor...")
+kw_model = KeyBERT()
+
+print("All models loaded successfully!")
+
+# Positive and negative word lists for fallback
+POSITIVE_WORDS = {
+    'love', '❤️', '💕', '💓', '💗', '💖', '💘', '💝',
+    'great', 'amazing', 'awesome', 'fantastic', 'wonderful',
+    'beautiful', 'perfect', 'excellent', 'brilliant',
+    'fan', 'favorite', 'favourite', 'best', 'good', 'nice',
+    'like', 'enjoy', 'appreciate', 'thank', 'thanks', 'legend'
+}
+
+NEGATIVE_WORDS = {
+    'hate', 'bad', 'terrible', 'awful', 'horrible', 'sucks',
+    'dislike', 'worst', 'poor', 'disappointing', 'waste',
+    'boring', 'useless', 'trash', 'garbage', 'cringe',
+    'overrated', 'hated', 'annoying', 'stupid', 'dumb'
+}
+
+def safe_truncate(text, max_length=512):
+    """Safely truncate text to max_length characters"""
+    if not text:
+        return ""
+    if len(text) > max_length:
+        return text[:max_length]
+    return text
+
+def clean_text(text: str) -> tuple:
+    """Clean and normalize text"""
+    if not text:
+        return "", ""
+    text = ' '.join(text.split())
+    text_lower = text.lower()
+    return text, text_lower
+
+def predict_sentiment(text: str) -> str:
+    """
+    FIXED: Unbiased sentiment prediction
+    Let the model decide without forcing positive/neutral
+    """
+    try:
+        if not text or len(text.strip()) < 2:
+            return "NEUTRAL"
+        
+        # Clean and truncate
+        text = safe_truncate(text, 512)
+        original_text = text
+        text, text_lower = clean_text(text)
+        
+        # Get model prediction (primary source)
+        try:
+            result = sentiment_model(text)[0]
+            model_label = result['label']
+            model_score = result['score']
+            
+            # Map model labels to our categories
+            # LABEL_0 = Negative, LABEL_1 = Neutral, LABEL_2 = Positive
+            if model_label == "LABEL_0":
+                return "NEGATIVE"
+            elif model_label == "LABEL_2":
+                return "POSITIVE"
+            elif model_label == "LABEL_1":
+                # Only return neutral if confidence is high
+                if model_score > 0.8:
+                    return "NEUTRAL"
+                # Otherwise, check keywords to decide
+                pass
+                
+        except Exception as model_err:
+            print(f"Model error: {model_err}")
+            # Fall through to keyword analysis
+        
+        # Keyword-based analysis (fallback only)
+        pos_count = sum(1 for word in POSITIVE_WORDS if word in text_lower)
+        neg_count = sum(1 for word in NEGATIVE_WORDS if word in text_lower)
+        
+        # Simple majority rule for keywords
+        if pos_count > neg_count and pos_count > 0:
+            return "POSITIVE"
+        elif neg_count > pos_count and neg_count > 0:
+            return "NEGATIVE"
+        
+        # Default to neutral only if absolutely no signal
+        return "NEUTRAL"
+        
+    except Exception as e:
+        print(f"Error in sentiment analysis: {e}")
+        return "NEUTRAL"
+
+def detect_sarcasm(text: str) -> str:
+    """Detect sarcasm in comment"""
+    try:
+        if not text:
+            return "NO"
+        
+        text = safe_truncate(text, 512)
+        if len(text.strip()) < 3:
+            return "NO"
+        
+        result = sarcasm_model(text)[0]
+        # LABEL_1 = sarcastic
+        return "YES" if result['label'] == "LABEL_1" and result['score'] > 0.55 else "NO"
+    except Exception as e:
+        return "NO"
+
+def detect_emotion(text: str) -> str:
+    """Detect emotion in comment"""
+    try:
+        if not text:
+            return "neutral"
+        
+        # Emoji-based fast detection
+        if '😭' in text or '😢' in text:
+            return "sadness"
+        elif '😊' in text or '😍' in text or '🥰' in text:
+            return "joy"
+        elif '😂' in text or '🤣' in text:
+            return "amusement"
+        elif '❤️' in text or '💕' in text:
+            return "love"
+        elif '🎉' in text or '🎊' in text:
+            return "excitement"
+        elif '😠' in text or '🤬' in text:
+            return "anger"
+        elif '😨' in text or '😱' in text:
+            return "fear"
+        
+        # Model-based detection
+        text = safe_truncate(text, 512)
+        if len(text.strip()) < 3:
+            return "neutral"
+        
+        result = emotion_model(text)[0]
+        return result['label']
+    except Exception as e:
+        return "neutral"
+
+def generate_summary(texts: list) -> str:
+    """Generate summary of all comments"""
+    try:
+        if not texts:
+            return "No comments to summarize"
+        
+        sample_size = min(100, len(texts))
+        combined = " ".join(texts[:sample_size])
+        combined = safe_truncate(combined, 1024)
+        
+        if len(combined) < 50:
+            return "Not enough comments to generate summary"
+        
+        summary = summarizer(combined, max_length=150, min_length=40, do_sample=False)
+        return summary[0]['summary_text']
+    except Exception as e:
+        print(f"Error in summary generation: {e}")
+        return "Summary generation failed"
+
+def extract_keywords(texts: list, top_n=15):
+    """Extract keywords from comments"""
+    try:
+        if not texts:
+            return []
+        
+        sample_size = min(200, len(texts))
+        combined = " ".join(texts[:sample_size])
+        combined = safe_truncate(combined, 2000)
+        
+        if len(combined) < 20:
+            return []
+        
+        keywords = kw_model.extract_keywords(
+            combined, 
+            keyphrase_ngram_range=(1, 2), 
+            stop_words='english', 
+            top_n=top_n
+        )
+        return [kw[0] for kw in keywords if kw and kw[0]]
+    except Exception as e:
+        print(f"Error in keyword extraction: {e}")
+        # Fallback: simple word frequency
+        words = combined.lower().split()
+        stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 
+                     'to', 'for', 'of', 'in', 'on', 'at', 'by', 'with', 'without', 'i', 
+                     'you', 'he', 'she', 'it', 'we', 'they', 'this', 'that', 'these', 'those'}
+        word_freq = {}
+        for word in words:
+            word = word.strip('.,!?;:()[]{}"\'')
+            if len(word) > 2 and word not in stop_words and not word.isdigit():
+                word_freq[word] = word_freq.get(word, 0) + 1
+        sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:top_n]
+        return [word for word, count in sorted_words]
+
+def process_comments_in_batches(comments, batch_size=100):
+    """Process comments in batches"""
+    total = len(comments)
+    
+    if total == 0:
+        return
+    
+    print(f"Processing {total} comments in batches of {batch_size}...")
+    
+    for i in range(0, total, batch_size):
+        batch = comments[i:i+batch_size]
+        batch_results = []
+        
+        for comment in batch:
+            try:
+                if not comment or len(comment.strip()) < 2:
+                    batch_results.append({
+                        "text": comment if comment else "",
+                        "sentiment": "NEUTRAL",
+                        "sarcasm": "NO",
+                        "emotion": "neutral"
+                    })
+                    continue
+                
+                sentiment = predict_sentiment(comment)
+                sarcasm = detect_sarcasm(comment)
+                emotion = detect_emotion(comment)
+                
+                batch_results.append({
+                    "text": comment,
+                    "sentiment": sentiment,
+                    "sarcasm": sarcasm,
+                    "emotion": emotion
+                })
+            except Exception as e:
+                batch_results.append({
+                    "text": comment if comment else "",
+                    "sentiment": "NEUTRAL",
+                    "sarcasm": "NO",
+                    "emotion": "unknown"
+                })
+        
+        yield batch_results
+        
+        if (i // batch_size + 1) % 10 == 0 or (i + batch_size) >= total:
+            processed = min(i + batch_size, total)
+            print(f"  Processed batch {i//batch_size + 1}/{(total + batch_size - 1)//batch_size} ({processed}/{total} comments)")
+
+def get_batch_stats(results_batches):
+    """Aggregate statistics from batch results"""
+    stats = {"positive": 0, "neutral": 0, "negative": 0}
+    all_results = []
+    
+    for batch in results_batches:
+        for item in batch:
+            all_results.append(item)
+            if item["sentiment"] == "POSITIVE":
+                stats["positive"] += 1
+            elif item["sentiment"] == "NEGATIVE":
+                stats["negative"] += 1
+            else:
+                stats["neutral"] += 1
+    
+    return stats, all_results
+
+def get_sentiment_score(stats):
+    """Calculate overall sentiment score"""
+    total = stats["positive"] + stats["neutral"] + stats["negative"]
+    if total == 0:
+        return 0
+    return ((stats["positive"] - stats["negative"]) / total) * 100

requirements.txt

@@ -0,0 +1,14 @@
+fastapi==0.104.1
+uvicorn==0.27.0
+transformers==4.36.0
+torch==2.1.0
+keybert==0.8.0
+requests==2.31.0
+yt-dlp==2023.12.30
+python-multipart==0.0.6
+sentencepiece==0.1.99
+protobuf==3.20.3
+numpy==1.24.3
+sentence-transformers==2.2.2
+aiofiles==23.2.1
+huggingface-hub==0.19.4

test_api.py

@@ -0,0 +1,54 @@
+import requests
+
+API_KEY = "AIzaSyBwwLnHXKmL5VQhkKzumCS5r1Cbi3HdUro"
+video_id = "dQw4w9WgXcQ"  # Rick Astley - has many comments
+
+print("="*50)
+print("Testing YouTube API Key")
+print("="*50)
+print(f"API Key: {API_KEY[:10]}...{API_KEY[-10:]}")
+print(f"Video ID: {video_id}")
+print()
+
+# Test video info endpoint
+url = f"https://www.googleapis.com/youtube/v3/videos?part=statistics&id={video_id}&key={API_KEY}"
+response = requests.get(url)
+data = response.json()
+
+if "error" in data:
+    print("[ERROR] API Error:")
+    print(f"  Message: {data['error']['message']}")
+    print(f"  Code: {data['error']['code']}")
+    print(f"  Reason: {data['error']['errors'][0]['reason']}")
+    print()
+    print("Possible solutions:")
+    print("1. Enable YouTube Data API v3 in Google Cloud Console")
+    print("2. Check if API key is correct")
+    print("3. Make sure billing is enabled (even for free tier)")
+else:
+    print("[SUCCESS] API Key is valid!")
+    if 'items' in data and len(data['items']) > 0:
+        print(f"  Video found!")
+        comment_count = data['items'][0]['statistics'].get('commentCount', 0)
+        print(f"  Comment count: {comment_count}")
+        
+        # Test fetching comments
+        print()
+        print("Testing comment fetch...")
+        comments_url = f"https://www.googleapis.com/youtube/v3/commentThreads?part=snippet&videoId={video_id}&maxResults=5&key={API_KEY}"
+        comments_response = requests.get(comments_url)
+        comments_data = comments_response.json()
+        
+        if "items" in comments_data:
+            print(f"[SUCCESS] Successfully fetched {len(comments_data['items'])} comments!")
+            if len(comments_data['items']) > 0:
+                first_comment = comments_data['items'][0]['snippet']['topLevelComment']['snippet']['textDisplay']
+                print(f"  First comment: {first_comment[:100]}...")
+        else:
+            print("[WARNING] Could not fetch comments:")
+            if "error" in comments_data:
+                print(f"  {comments_data['error'].get('message', 'Unknown error')}")
+    else:
+        print("[ERROR] Video not found")
+
+print("="*50)