Restore e77a637: vtv_shorts.py

vtv_shorts.py

@@ -1,10 +1,5 @@
 """
 VTV Nam Bộ YouTube Shorts Scraper
-- Fetches channel ID from YouTube page
-- Uses YouTube RSS feed to get all videos
-- Filters for shorts (#shorts in title or /shorts/ in link)
-- Falls back to yt-dlp if RSS fails
-- Returns list of shorts with id, title, thumbnail
 """
 import requests
 import re
@@ -17,7 +12,7 @@ from xml.etree import ElementTree as ET
 
 _cache = {}
 _lock = threading.Lock()
-CACHE_TTL = 1800  # 30 min
+CACHE_TTL = 1800
 
 UA = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
@@ -34,13 +29,10 @@ def _set_cache(key, data):
     with _lock:
         _cache[key] = {'t': time.time(), 'd': data}
 
-
 def get_channel_id(username):
-    """Get YouTube channel ID from username by parsing the channel page."""
     cached = _cached(f'ch_id_{username}')
     if cached:
         return cached
-
     channel_id = None
     try:
         url = f"https://www.youtube.com/@{username}"
@@ -49,76 +41,49 @@ def get_channel_id(username):
             m = re.search(r'<meta\s+property="og:url"\s+content="https://www.youtube.com/channel/(UC[^"]+)"', r.text)
             if m:
                 channel_id = m.group(1)
-            if not channel_id:
-                m = re.search(r'<link\s+rel="canonical"\s+href="https://www.youtube.com/channel/(UC[^"]+)"', r.text)
-                if m:
-                    channel_id = m.group(1)
             if not channel_id:
                 m = re.search(r'"channelId":"(UC[^"]+)"', r.text)
                 if m:
                     channel_id = m.group(1)
-            if not channel_id:
-                m = re.search(r'"externalId":"(UC[^"]+)"', r.text)
-                if m:
-                    channel_id = m.group(1)
     except Exception as e:
         print(f"Error getting channel ID for @{username}: {e}")
-
     if channel_id:
         _set_cache(f'ch_id_{username}', channel_id)
     return channel_id
 
-
 def scrape_via_rss(channel_id, max_count=100):
-    """Scrape videos from YouTube RSS feed."""
     shorts = []
     try:
         url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
         r = requests.get(url, headers=UA, timeout=15)
         if r.status_code != 200:
             return shorts
-
         root = ET.fromstring(r.text)
         ns = {'atom': 'http://www.w3.org/2005/Atom', 'yt': 'http://www.youtube.com/xml/schemas/2015'}
-
         for entry in root.findall('atom:entry', ns)[:max_count]:
             title_el = entry.find('atom:title', ns)
             title = html_lib.unescape(title_el.text) if title_el is not None and title_el.text else ''
-
             vid_el = entry.find('yt:videoId', ns)
             vid = vid_el.text if vid_el is not None else ''
             if not vid:
                 continue
-
             is_short = '#shorts' in title.lower() or '#short' in title.lower()
-
             link_el = entry.find('atom:link', ns)
             link = link_el.get('href', '') if link_el is not None else ''
             if '/shorts/' in link:
                 is_short = True
-
             if is_short:
-                shorts.append({
-                    'id': vid,
-                    'title': title,
-                    'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
-                    'channel': 'vtvnambo',
-                })
+                shorts.append({'id': vid, 'title': title, 'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg", 'channel': 'vtvnambo'})
     except Exception as e:
         print(f"RSS scrape error: {e}")
-
     return shorts
 
-
 def scrape_via_ydlp(username, count=50):
-    """Scrape shorts using yt-dlp."""
     shorts = []
     try:
         url = f"https://www.youtube.com/@{username}/shorts"
         result = subprocess.run(
-            ["yt-dlp", "--dump-json", "--flat-playlist", "--no-download",
-             "--playlist-end", str(count),
-             url],
+            ["yt-dlp", "--dump-json", "--flat-playlist", "--no-download", "--playlist-end", str(count), url],
             capture_output=True, text=True, timeout=90
         )
         if result.returncode == 0 and result.stdout.strip():
@@ -134,71 +99,19 @@ def scrape_via_ydlp(username, count=50):
                         continue
                     seen_ids.add(vid)
                     title = entry.get('title', 'VTV Nam Bộ Short')
-                    shorts.append({
-                        'id': vid,
-                        'title': title,
-                        'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
-                        'channel': 'vtvnambo',
-                    })
+                    shorts.append({'id': vid, 'title': title, 'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg", 'channel': 'vtvnambo'})
                 except json.JSONDecodeError:
                     continue
     except (subprocess.TimeoutExpired, FileNotFoundError) as e:
         print(f"yt-dlp error: {e}")
-    except Exception as e:
-        print(f"yt-dlp error: {e}")
-
-    return shorts
-
-
-def scrape_via_ydlp_videos_filter(username, count=100):
-    """Scrape all videos and filter for shorts by duration."""
-    shorts = []
-    try:
-        url = f"https://www.youtube.com/@{username}/videos"
-        result = subprocess.run(
-            ["yt-dlp", "--dump-json", "--flat-playlist", "--no-download",
-             "--playlist-end", str(count),
-             "--match-filter", "duration < 65",
-             url],
-            capture_output=True, text=True, timeout=90
-        )
-        if result.returncode == 0 and result.stdout.strip():
-            seen_ids = set()
-            for line in result.stdout.strip().split('\n'):
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    entry = json.loads(line)
-                    vid = entry.get('id', '')
-                    if not vid or vid in seen_ids:
-                        continue
-                    seen_ids.add(vid)
-                    title = entry.get('title', 'VTV Nam Bộ Short')
-                    shorts.append({
-                        'id': vid,
-                        'title': title,
-                        'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
-                        'channel': 'vtvnambo',
-                    })
-                except json.JSONDecodeError:
-                    continue
-    except Exception as e:
-        print(f"yt-dlp videos filter error: {e}")
-
     return shorts
 
-
 def get_vtvnambo_shorts(max_count=50):
-    """Get all shorts from VTV Nam Bộ channel. Tries multiple methods."""
     cached = _cached('vtvnambo_shorts_v2')
     if cached is not None:
         return cached
-
     all_shorts = []
     seen_ids = set()
-
-    # Method 1: RSS feed (most reliable, no JS needed)
     channel_id = get_channel_id('vtvnambo')
     if channel_id:
         rss_shorts = scrape_via_rss(channel_id, max_count * 2)
@@ -206,89 +119,22 @@ def get_vtvnambo_shorts(max_count=50):
             if s['id'] not in seen_ids:
                 seen_ids.add(s['id'])
                 all_shorts.append(s)
-
-    # Method 2: yt-dlp on shorts page
     if len(all_shorts) < 3:
         ydlp_shorts = scrape_via_ydlp('vtvnambo', max_count)
         for s in ydlp_shorts:
             if s['id'] not in seen_ids:
                 seen_ids.add(s['id'])
                 all_shorts.append(s)
-
-    # Method 3: yt-dlp on videos page with duration filter
-    if len(all_shorts) < 3:
-        ydlp_vids = scrape_via_ydlp_videos_filter('vtvnambo', max_count * 2)
-        for s in ydlp_vids:
-            if s['id'] not in seen_ids:
-                seen_ids.add(s['id'])
-                all_shorts.append(s)
-
-    # Method 4: requests + regex on shorts page
-    if len(all_shorts) < 3:
-        try:
-            url = "https://www.youtube.com/@vtvnambo/shorts"
-            r = requests.get(url, headers=UA, timeout=15)
-            if r.status_code == 200:
-                for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"', r.text):
-                    vid = m.group(1)
-                    if vid in seen_ids:
-                        continue
-                    seen_ids.add(vid)
-                    snip = r.text[max(0, m.start() - 900):m.start() + 1600]
-                    title = ""
-                    mt = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', snip)
-                    if not mt:
-                        mt = re.search(r'"accessibilityText":"([^"]+)"', snip)
-                    if mt:
-                        title = html_lib.unescape(mt.group(1)).replace('\n', ' ').strip()
-                    if not title:
-                        title = "VTV Nam Bộ Short"
-                    all_shorts.append({
-                        'id': vid,
-                        'title': title,
-                        'img': f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg",
-                        'channel': 'vtvnambo',
-                    })
-        except Exception as e:
-            print(f"Requests regex error: {e}")
-
     result = all_shorts[:max_count]
     _set_cache('vtvnambo_shorts_v2', result)
     return result
 
-
-# Alias for backward compatibility (fix the typo import in main.py)
 get_vtvnamo_shorts = get_vtvnambo_shorts
 
-
 def get_wc_related_shorts(max_count=30):
-    """Get World Cup / football related shorts from VTV Nam Bộ."""
     all_shorts = get_vtvnambo_shorts(max_count * 3)
-
-    wc_kws = [
-        'world cup', 'wc 2026', 'worldcup', 'fifa', 'bóng đá',
-        'trận đấu', 'đội tuyển', 'tuyển', 'vòng loại',
-        'khoảnh khắc', 'highlights', 'bàn thắng', 'goal',
-        'kết quả', 'tỉ số', 'việt nam', 'vn',
-        'ngoại hạng', 'premier league', 'champions league',
-        'laliga', 'serie a', 'bundesliga', 'ligue 1',
-        'copa', 'europa', 'c1', 'c2',
-        'messi', 'ronaldo', 'neymar', 'mbappe', 'haaland',
-        'v-league', 'vleague', 'bóng đá việt',
-        'đội bóng', 'hlv', 'huấn luyện viên',
-        'chuyển nhượng', 'transfer',
-        'asian cup', 'aff cup', 'sea games',
-        'olympic', 'u23', 'u20', 'u17',
-    ]
-
-    wc_shorts = []
-    for s in all_shorts:
-        tl = s.get('title', '').lower()
-        if any(k in tl for k in wc_kws):
-            wc_shorts.append(s)
-
-    # If no WC-specific shorts found, return all VTV shorts
+    wc_kws = ['world cup', 'wc 2026', 'worldcup', 'fifa', 'bóng đá', 'trận đấu', 'đội tuyển']
+    wc_shorts = [s for s in all_shorts if any(k in s.get('title', '').lower() for k in wc_kws)]
     if not wc_shorts:
         wc_shorts = all_shorts
-
     return wc_shorts[:max_count]