Update services/aggregator.py

services/aggregator.py

@@ -1,75 +1,119 @@
 """
 services/aggregator.py
-Kumpulkan data dari YouTube, Reddit, dan Instagram (Apify).
+Kumpulkan data dari:
+  1. YouTube         (Google API)
+  2. Reddit          (PRAW)
+  3. Instagram       (Apify)
+  4. TikTok          (Apify)
+  5. Google News     (SerpApi)
 """
 
 from services.youtube import search_videos, get_comments
 from services.reddit import get_reddit_comments
 from services.preprocessing import clean_text, is_valid
 
-# Instagram via Apify — opsional, skip jika API key tidak ada
 try:
     from services.instagram import get_instagram_data
-    INSTAGRAM_AVAILABLE = True
+    INSTAGRAM_OK = True
 except ImportError:
-    INSTAGRAM_AVAILABLE = False
+    INSTAGRAM_OK = False
     def get_instagram_data(kw): return []
 
+try:
+    from services.tiktok import get_tiktok_data
+    TIKTOK_OK = True
+except ImportError:
+    TIKTOK_OK = False
+    def get_tiktok_data(kw): return []
+
+try:
+    from services.google_news import get_google_news
+    GNEWS_OK = True
+except ImportError:
+    GNEWS_OK = False
+    def get_google_news(kw): return []
+
 
 def collect_data(keyword: str, source: str = "all") -> list[tuple[str, str]]:
     """
     Return: list of (source_label, cleaned_text)
 
-    source options:
-        "all"       → YouTube + Reddit + Instagram
-        "youtube"   → YouTube saja
-        "reddit"    → Reddit saja
-        "instagram" → Instagram saja
-        kombinasi   → "youtube,instagram" dst.
+    source values:
+        "all"              → semua 5 platform
+        "youtube"          → YouTube saja
+        "reddit"           → Reddit saja
+        "instagram"        → Instagram saja
+        "tiktok"           → TikTok saja
+        "news"             → Google News saja
+        kombinasi CSV      → "youtube,tiktok" / "tiktok,news" / dst.
     """
-    all_data = []
+    all_data: list[tuple[str, str]] = []
     src = source.lower()
 
-    # ── YOUTUBE ──
-    if src in ("all", "youtube") or "youtube" in src:
+    def wants(platform: str) -> bool:
+        return src == "all" or platform in src
+
+    # 1. YOUTUBE
+    if wants("youtube"):
+        before = len(all_data)
         try:
-            vids = search_videos(keyword)
-            for vid in vids:
+            for vid in search_videos(keyword):
                 for c in get_comments(vid):
                     all_data.append(("youtube", c))
+            print(f"✅ YouTube: {len(all_data)-before} komentar")
         except Exception as e:
-            print(f"⚠️  YouTube collect error: {e}")
+            print(f"⚠️  YouTube error: {e}")
 
-    # ── REDDIT ──
-    if src in ("all", "reddit") or "reddit" in src:
+    # 2. REDDIT
+    if wants("reddit"):
+        before = len(all_data)
         try:
             for c in get_reddit_comments(keyword):
                 all_data.append(("reddit", c))
+            print(f"✅ Reddit: {len(all_data)-before} komentar")
+        except Exception as e:
+            print(f"⚠️  Reddit error: {e}")
+
+    # 3. INSTAGRAM
+    if wants("instagram") and INSTAGRAM_OK:
+        before = len(all_data)
+        try:
+            for text in get_instagram_data(keyword):
+                all_data.append(("instagram", text))
+            print(f"✅ Instagram: {len(all_data)-before} teks")
+        except Exception as e:
+            print(f"⚠️  Instagram error: {e}")
+
+    # 4. TIKTOK
+    if wants("tiktok") and TIKTOK_OK:
+        before = len(all_data)
+        try:
+            for text in get_tiktok_data(keyword):
+                all_data.append(("tiktok", text))
+            print(f"✅ TikTok: {len(all_data)-before} teks")
         except Exception as e:
-            print(f"⚠️  Reddit collect error: {e}")
+            print(f"⚠️  TikTok error: {e}")
 
-    # ── INSTAGRAM ──
-    if src in ("all", "instagram") or "instagram" in src:
-        if INSTAGRAM_AVAILABLE:
-            try:
-                for text in get_instagram_data(keyword):
-                    all_data.append(("instagram", text))
-            except Exception as e:
-                print(f"⚠️  Instagram collect error: {e}")
-        else:
-            print("⚠️  Instagram scraper tidak tersedia")
+    # 5. GOOGLE NEWS
+    if wants("news") and GNEWS_OK:
+        before = len(all_data)
+        try:
+            for text in get_google_news(keyword):
+                all_data.append(("news", text))
+            print(f"✅ Google News: {len(all_data)-before} teks")
+        except Exception as e:
+            print(f"⚠️  Google News error: {e}")
 
-    # ── FALLBACK jika semua kosong ──
+    # FALLBACK
     if not all_data:
         print("⚠️  Tidak ada data dari semua sumber")
         all_data = [("unknown", "data tidak ditemukan")]
 
-    # ── CLEAN & FILTER ──
+    # CLEAN & FILTER
     cleaned = [
         (src_label, clean_text(text))
         for src_label, text in all_data
         if is_valid(text)
     ]
-
-    print(f"✅ Total data terkumpul: {len(cleaned)} dari {len(all_data)} raw")
+    print(f"✅ Total: {len(cleaned)} teks dari {len(all_data)} raw")
     return cleaned