Upload yt_scraper.py

yt_scraper.py

@@ -57,179 +57,89 @@ def run_yt_dlp(args, timeout=120):
 def get_channel_shorts_via_playlist(channel_username, max_count=100):
     """Get shorts from channel's shorts page using yt-dlp"""
     shorts = []
-    
-    # Method 1: Fetch from /shorts page
     url = f"https://www.youtube.com/@{channel_username}/shorts"
     items = run_yt_dlp([
-        "--dump-json",
-        "--flat-playlist",
-        "--no-download",
+        "--dump-json", "--flat-playlist", "--no-download",
         "--playlist-end", str(max_count),
         "--no-check-certificates",
         "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
         url
     ], timeout=90)
-    
     seen_ids = set()
     for item in items:
         vid = item.get('id', '')
-        if not vid or vid in seen_ids:
-            continue
+        if not vid or vid in seen_ids: continue
         seen_ids.add(vid)
-        
         title = item.get('title', 'VTV Nam Bộ Short')
         duration = item.get('duration', 0) or 0
-        
-        # Only include actual shorts (<= 120s to be safe)
         if duration <= 120 or '#shorts' in title.lower() or '#short' in title.lower():
-            shorts.append({
-                'id': vid,
-                'title': title,
-                'duration': duration,
-                'channel': channel_username,
-                'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
-            })
-    
+            shorts.append({'id': vid, 'title': title, 'duration': duration, 'channel': channel_username, 'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"})
     return shorts
 
 def get_channel_videos_filter_shorts(channel_username, max_count=200):
     """Get all videos from /videos page and filter for shorts by duration"""
     shorts = []
-    
     url = f"https://www.youtube.com/@{channel_username}/videos"
     items = run_yt_dlp([
-        "--dump-json",
-        "--flat-playlist",
-        "--no-download",
+        "--dump-json", "--flat-playlist", "--no-download",
         "--playlist-end", str(max_count),
         "--match-filter", "duration > 0 and duration <= 120",
         "--no-check-certificates",
         "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
         url
     ], timeout=120)
-    
     seen_ids = set()
     for item in items:
         vid = item.get('id', '')
-        if not vid or vid in seen_ids:
-            continue
+        if not vid or vid in seen_ids: continue
         seen_ids.add(vid)
-        
-        title = item.get('title', 'VTV Nam Bộ Short')
-        duration = item.get('duration', 0) or 0
-        
-        shorts.append({
-            'id': vid,
-            'title': title,
-            'duration': duration,
-            'channel': channel_username,
-            'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
-        })
-    
+        shorts.append({'id': vid, 'title': item.get('title', ''), 'duration': item.get('duration', 0) or 0, 'channel': channel_username, 'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"})
     return shorts
 
 def get_shorts_with_direct_url(video_ids):
     """Get direct video download URLs for given video IDs"""
     results = []
-    
-    for vid in video_ids[:20]:  # Limit to avoid timeout
+    for vid in video_ids[:20]:
         try:
             url = f"https://www.youtube.com/shorts/{vid}"
-            items = run_yt_dlp([
-                "--dump-json",
-                "--no-download",
-                "--no-check-certificates",
-                "--format", "best[filesize<10M]/best",
-                url
-            ], timeout=30)
-            
+            items = run_yt_dlp(["--dump-json", "--no-download", "--no-check-certificates", "--format", "best[filesize<10M]/best", url], timeout=30)
             if items:
                 info = items[0]
                 direct_url = info.get('url', '')
                 if not direct_url:
-                    # Try to get from formats
                     formats = info.get('formats', [])
                     for f in formats:
                         if f.get('vcodec') != 'none' and f.get('acodec') != 'none':
                             direct_url = f.get('url', '')
                             break
-                
                 if direct_url:
-                    results.append({
-                        'id': vid,
-                        'title': info.get('title', ''),
-                        'direct_url': direct_url,
-                        'thumbnail': info.get('thumbnail', f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"),
-                        'duration': info.get('duration', 0),
-                    })
+                    results.append({'id': vid, 'title': info.get('title', ''), 'direct_url': direct_url, 'thumbnail': info.get('thumbnail', f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"), 'duration': info.get('duration', 0)})
         except Exception as e:
             print(f"Error getting URL for {vid}: {e}")
-    
     return results
 
 def get_vtvnambo_shorts(max_count=50):
     """Get all shorts from VTV Nam Bộ using yt-dlp"""
     cached = _cached('vtvnambo_shorts_yt')
-    if cached is not None:
-        return cached
-
-    all_shorts = []
-    seen_ids = set()
-
-    # Method 1: /shorts page
-    print(f"[yt-dlp] Fetching /shorts page...")
+    if cached is not None: return cached
+    all_shorts = []; seen_ids = set()
     shorts_page = get_channel_shorts_via_playlist('vtvnambo', max_count)
     for s in shorts_page:
-        if s['id'] not in seen_ids:
-            seen_ids.add(s['id'])
-            all_shorts.append(s)
-    print(f"[yt-dlp] /shorts page: {len(shorts_page)} shorts")
-
-    # Method 2: /videos page with duration filter
+        if s['id'] not in seen_ids: seen_ids.add(s['id']); all_shorts.append(s)
     if len(all_shorts) < 5:
-        print(f"[yt-dlp] Fetching /videos page with filter...")
         videos_filtered = get_channel_videos_filter_shorts('vtvnambo', max_count * 2)
         for s in videos_filtered:
-            if s['id'] not in seen_ids:
-                seen_ids.add(s['id'])
-                all_shorts.append(s)
-        print(f"[yt-dlp] /videos filter: {len(videos_filtered)} shorts")
-
+            if s['id'] not in seen_ids: seen_ids.add(s['id']); all_shorts.append(s)
     result = all_shorts[:max_count]
     _set_cache('vtvnambo_shorts_yt', result)
-    print(f"[yt-dlp] Total: {len(result)} shorts from VTV Nam Bộ")
     return result
 
 def get_wc_related_shorts(max_count=30):
     """Get World Cup / football related shorts"""
     all_shorts = get_vtvnambo_shorts(max_count * 3)
-    
-    wc_kws = [
-        'world cup', 'wc 2026', 'worldcup', 'fifa', 'bóng đá',
-        'trận đấu', 'đội tuyển', 'tuyển', 'vòng loại',
-        'khoảnh khắc', 'highlights', 'bàn thắng', 'goal',
-        'kết quả', 'tỉ số', 'việt nam', 'vn',
-        'ngoại hạng', 'premier league', 'champions league',
-        'laliga', 'serie a', 'bundesliga', 'ligue 1',
-        'copa', 'europa', 'c1', 'c2',
-        'messi', 'ronaldo', 'neymar', 'mbappe', 'haaland',
-        'v-league', 'vleague', 'bóng đá việt',
-        'đội bóng', 'hlv', 'huấn luyện viên',
-        'chuyển nhượng', 'transfer',
-        'asian cup', 'aff cup', 'sea games',
-        'olympic', 'u23', 'u20', 'u17',
-    ]
-    
-    wc_shorts = []
-    for s in all_shorts:
-        tl = s.get('title', '').lower()
-        if any(k in tl for k in wc_kws):
-            wc_shorts.append(s)
-    
-    if not wc_shorts:
-        wc_shorts = all_shorts
-    
+    wc_kws = ['world cup', 'wc 2026', 'worldcup', 'fifa', 'bóng đá', 'trận đấu', 'đội tuyển', 'tuyển', 'vòng loại', 'khoảnh khắc', 'highlights', 'bàn thắng', 'goal', 'kết quả', 'tỉ số', 'việt nam', 'vn', 'ngoại hạng', 'premier league', 'champions league', 'laliga', 'serie a', 'bundesliga', 'ligue 1', 'copa', 'europa', 'c1', 'c2', 'messi', 'ronaldo', 'neymar', 'mbappe', 'haaland', 'v-league', 'vleague', 'bóng đá việt', 'đội bóng', 'hlv', 'huấn luyện viên', 'chuyển nhượng', 'transfer', 'asian cup', 'aff cup', 'sea games', 'olympic', 'u23', 'u20', 'u17']
+    wc_shorts = [s for s in all_shorts if any(k in s.get('title','').lower() for k in wc_kws)]
+    if not wc_shorts: wc_shorts = all_shorts
     return wc_shorts[:max_count]
 
-# Aliases
 get_vtvnamo_shorts = get_vtvnambo_shorts