Upload yt_scraper.py

yt_scraper.py

@@ -1,141 +1,234 @@
 """
-YouTube Shorts Scraper using requests HTML parsing (no yt-dlp dependency)
-Optimized for fast load with cache + instant fallback to prevent homepage blocking
+YouTube Shorts Scraper using yt-dlp (already installed on Space)
+Runs yt-dlp as subprocess to extract video info and direct URLs
 """
-import requests
+import subprocess
 import json
 import time
 import threading
-import html as html_lib
+import os
+import re as re_mod
 
 _cache = {}
 _lock = threading.Lock()
-CACHE_TTL = 1800  # 30 min cache - longer to reduce timeout issues
-
-HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Accept-Language":"vi,en;q=0.8"}
-
-# Static fallback - always available instantly
-_SHORTS_FALLBACK = {
-    'baodantri7941': [
-        {"id":"Lu_iCQ5YwNM","title":"Công an lập hồ sơ xử lý người phụ nữ chửi bới tát nam tài xế ô tô ở Hà Nội"},
-        {"id":"CwWvijF8BOA","title":"Chú rể Ninh Bình bật khóc nhận món quà bí mật người cha quá cố gửi 26 năm trước"},
-        {"id":"tvPewsc2ph4","title":"Tính năng ẩn trên iPhone giúp giảm mỏi mắt"},
-        {"id":"b1Nxzv9ixlU","title":"Y án 3 năm tù với nữ tài xế uống 8 lon bia lái xe tông chủ tịch xã tử vong"},
-        {"id":"Xp5eTwAZAis","title":"Người đánh hàng xóm tại chung cư ở Hà Nội bị tuyên hơn 4 tháng tù"},
-    ],
-    'baosuckhoedoisongboyte': [
-        {"id":"7Pd6vZ2Lz1M","title":"Hành động ấm lòng của người đàn ông tìm kiếm 5 học sinh tử vong ở sông Lô"},
-        {"id":"SlHLt_ZyPiE","title":"Xử phạt người đàn ông xóa số điện thoại cứu hộ trên cao tốc Bắc Nam"},
-        {"id":"IUOprcJyYr4","title":"Phụ nữ táo bón có phải do lười ăn rau?"},
-        {"id":"YY8ojFNE-AU","title":"Quái xế tự quay clip nẹt pô đánh võng đăng TikTok bị xử lý"},
-    ]
-}
+CACHE_TTL = 600  # 10 min cache
 
 def _cached(key):
-    """Get cached data if still valid"""
     with _lock:
-        if key in _cache:
-            entry = _cache[key]
-            if time.time() - entry['t'] < CACHE_TTL:
-                return entry['d']
+        if key in _cache and time.time() - _cache[key]['t'] < CACHE_TTL:
+            return _cache[key]['d']
     return None
 
 def _set_cache(key, data):
-    """Set cache with timestamp"""
     with _lock:
         _cache[key] = {'t': time.time(), 'd': data}
 
-def _get_channel_shorts_requests(channel, count=15):
-    """Fast requests-based scraping with timeout protection"""
+def run_yt_dlp(args, timeout=120):
+    """Run yt-dlp and return parsed JSON lines"""
     try:
-        url=f"https://www.youtube.com/@{channel}/shorts"
-        r=requests.get(url, headers=HEADERS, timeout=6)  # Fast 6s timeout
-        if r.status_code!=200:
-            return None
-        html=r.text; ids=[]; items=[]
-        import re
-        for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
-            vid=m.group(1)
-            if vid in ids: continue
-            ids.append(vid)
-            snip=html[max(0,m.start()-900):m.start()+1600]
-            title=""
-            mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip)
-            if not mt: mt=re.search(r'"accessibilityText":"([^"]+)"',snip)
-            if mt: title=html_lib.unescape(mt.group(1)).replace('\n',' ').strip()
-            if not title: title="YouTube Short"
-            items.append({"id":vid,"title":title,"channel":channel})
-            if len(items)>=count: break
+        result = subprocess.run(
+            ["yt-dlp"] + args,
+            capture_output=True, text=True, timeout=timeout
+        )
+        if result.returncode != 0 and not result.stdout.strip():
+            print(f"yt-dlp error: {result.stderr[:200]}")
+            return []
+        lines = result.stdout.strip().split('\n')
+        items = []
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                items.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
         return items
-    except Exception:
-        return None
+    except subprocess.TimeoutExpired:
+        print("yt-dlp timeout")
+        return []
+    except FileNotFoundError:
+        print("yt-dlp not found!")
+        return []
+    except Exception as e:
+        print(f"yt-dlp exception: {e}")
+        return []
 
-def get_channel_shorts_fast(channel_username, max_count=25):
-    """Get shorts fast - requests with fallback"""
-    import re
-    
-    # Try fast requests scraping
-    result = _get_channel_shorts_requests(channel_username, max_count)
+def get_channel_shorts_via_playlist(channel_username, max_count=100):
+    """Get shorts from channel's shorts page using yt-dlp"""
+    shorts = []
     
-    # If success, add image URL
-    if result:
-        for item in result:
-            item['img'] = f"https://i.ytimg.com/vi/{item['id']}/hqdefault.jpg"
-        return result
+    # Method 1: Fetch from /shorts page
+    url = f"https://www.youtube.com/@{channel_username}/shorts"
+    items = run_yt_dlp([
+        "--dump-json",
+        "--flat-playlist",
+        "--no-download",
+        "--playlist-end", str(max_count),
+        "--no-check-certificates",
+        "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        url
+    ], timeout=90)
     
-    # Return static fallback
-    fb = _SHORTS_FALLBACK.get(channel_username, [])
-    return [{"id":v["id"],"title":v["title"],"channel":channel_username,"img":f"https://i.ytimg.com/vi/{v['id']}/hqdefault.jpg"} for v in fb]
-
-def get_dantri_shorts(max_count=25):
-    """Get Dantri shorts - separate cache, fast fallback"""
-    cached = _cached('dantri_shorts_yt')
-    if cached is not None:
-        return cached
+    seen_ids = set()
+    for item in items:
+        vid = item.get('id', '')
+        if not vid or vid in seen_ids:
+            continue
+        seen_ids.add(vid)
+        
+        title = item.get('title', 'VTV Nam Bộ Short')
+        duration = item.get('duration', 0) or 0
+        
+        # Only include actual shorts (<= 120s to be safe)
+        if duration <= 120 or '#shorts' in title.lower() or '#short' in title.lower():
+            shorts.append({
+                'id': vid,
+                'title': title,
+                'duration': duration,
+                'channel': channel_username,
+                'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
+            })
     
-    shorts = get_channel_shorts_fast('baodantri7941', max_count)
-    _set_cache('dantri_shorts_yt', shorts)
     return shorts
 
-def get_skds_shorts(max_count=25):
-    """Get SKĐS shorts - separate cache, fast fallback"""
-    cached = _cached('skds_shorts_yt')
-    if cached is not None:
-        return cached
+def get_channel_videos_filter_shorts(channel_username, max_count=200):
+    """Get all videos from /videos page and filter for shorts by duration"""
+    shorts = []
+    
+    url = f"https://www.youtube.com/@{channel_username}/videos"
+    items = run_yt_dlp([
+        "--dump-json",
+        "--flat-playlist",
+        "--no-download",
+        "--playlist-end", str(max_count),
+        "--match-filter", "duration > 0 and duration <= 120",
+        "--no-check-certificates",
+        "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        url
+    ], timeout=120)
+    
+    seen_ids = set()
+    for item in items:
+        vid = item.get('id', '')
+        if not vid or vid in seen_ids:
+            continue
+        seen_ids.add(vid)
+        
+        title = item.get('title', 'VTV Nam Bộ Short')
+        duration = item.get('duration', 0) or 0
+        
+        shorts.append({
+            'id': vid,
+            'title': title,
+            'duration': duration,
+            'channel': channel_username,
+            'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
+        })
     
-    shorts = get_channel_shorts_fast('baosuckhoedoisongboyte', max_count)
-    _set_cache('skds_shorts_yt', shorts)
     return shorts
 
-def get_vtvnambo_shorts(max_count=50):
-    """Get interleaved Dantri + SKĐS shorts - optimized with separate caching"""
-    dantri = get_dantri_shorts(max_count // 2 + 10)
-    skds = get_skds_shorts(max_count // 2 + 10)
-    
-    # Interleave them
-    result = []
-    seen = set()
-    i, j = 0, 0
+def get_shorts_with_direct_url(video_ids):
+    """Get direct video download URLs for given video IDs"""
+    results = []
     
-    while (i < len(dantri) or j < len(skds)) and len(result) < max_count:
-        if i < len(dantri):
-            item = dantri[i]
-            if item.get('id') not in seen:
-                seen.add(item.get('id'))
-                result.append(item)
-            i += 1
-        if j < len(skds):
-            item = skds[j]
-            if item.get('id') not in seen:
-                seen.add(item.get('id'))
-                result.append(item)
-            j += 1
+    for vid in video_ids[:20]:  # Limit to avoid timeout
+        try:
+            url = f"https://www.youtube.com/shorts/{vid}"
+            items = run_yt_dlp([
+                "--dump-json",
+                "--                "--no-check-certificates",
+                "--format", "best[filesize<10M]/best",
+                url
+            ], timeout=30)
+            
+            if items:
+                info = items[0]
+                direct_url = info.get('url', '')
+                if not direct_url:
+                    # Try to get from formats
+                    formats = info.get('formats', [])
+                    for f in formats:
+                        if f.get('vcodec') != 'none' and f.get('acodec') != 'none':
+                            direct_url = f.get('url', '')
+                            break
+                
+                if direct_url:
+                    results.append({
+                        'id': vid,
+                        'title': info.get('title', ''),
+                        'direct_url': direct_url,
+                        'thumbnail': info.get('thumbnail', f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"),
+                        'duration': info.get('duration', 0),
+                    })
+        except Exception as e:
+            print(f"Error getting URL for {vid}: {e}")
     
+    return results
+
+def get_vtvnambo_shorts(max_count=50):
+    """Get all shorts from VTV Nam Bộ using yt-dlp"""
+    cached = _cached('vtvnambo_shorts_yt')
+    if cached is not None:
+        return cached
+
+    all_shorts = []
+    seen_ids = set()
+
+    # Method 1: /shorts page
+    print(f"[yt-dlp] Fetching /shorts page...")
+    shorts_page = get_channel_shorts_via_playlist('vtvnambo', max_count)
+    for s in shorts_page:
+        if s['id'] not in seen_ids:
+            seen_ids.add(s['id'])
+            all_shorts.append(s)
+    print(f"[yt-dlp] /shorts page: {len(shorts_page)} shorts")
+
+    # Method 2: /videos page with duration filter
+    if len(all_shorts) < 5:
+        print(f"[yt-dlp] Fetching /videos page with filter...")
+        videos_filtered = get_channel_videos_filter_shorts('vtvnambo', max_count * 2)
+        for s in videos_filtered:
+            if s['id'] not in seen_ids:
+                seen_ids.add(s['id'])
+                all_shorts.append(s)
+        print(f"[yt-dlp] /videos filter: {len(videos_filtered)} shorts")
+
+    result = all_shorts[:max_count]
+    _set_cache('vtvnambo_shorts_yt', result)
+    print(f"[yt-dlp] Total: {len(result)} shorts from VTV Nam Bộ")
     return result
 
 def get_wc_related_shorts(max_count=30):
-    """Get World Cup / football related shorts - now uses same cache as main shorts"""
-    return get_vtvnambo_shorts(max_count)
+    """Get World Cup / football related shorts"""
+    all_shorts = get_vtvnambo_shorts(max_count * 3)
+    
+    wc_kws = [
+        'world cup', 'wc 2026', 'worldcup', 'fifa', 'bóng đá',
+        'trận đấu', 'đội tuyển', 'tuyển', 'vòng loại',
+        'khoảnh khắc', 'highlights', 'bàn thắng', 'goal',
+        'kết quả', 'tỉ số', 'việt nam', 'vn',
+        'ngoại hạng', 'premier league', 'champions league',
+        'laliga', 'serie a', 'bundesliga', 'ligue 1',
+        'copa', 'europa', 'c1', 'c2',
+        'messi', 'ronaldo', 'neymar', 'mbappe', 'haaland',
+        'v-league', 'vleague', 'bóng đá việt',
+        'đội bóng', 'hlv', 'huấn luyện viên',
+        'chuyển nhượng', 'transfer',
+        'asian cup', 'aff cup', 'sea games',
+        'olympic', 'u23', 'u20', 'u17',
+    ]
+    
+    wc_shorts = []
+    for s in all_shorts:
+        tl = s.get('title', '').lower()
+        if any(k in tl for k in wc_kws):
+            wc_shorts.append(s)
+    
+    if not wc_shorts:
+        wc_shorts = all_shorts
+    
+    return wc_shorts[:max_count]
 
-# Alias for backward compatibility
+# Aliases
 get_vtvnamo_shorts = get_vtvnambo_shorts