Fix VTV scraper: add VTV6/VTV10, scrape xemtv PHP for fresh URLs

vtv_scraper.py

@@ -1,10 +1,9 @@
 """
 VTV Channels Scraper
-Fetches stream URLs from hd.xemtv.net for VTV1-VTV9 + VTV Cần Thơ
-Uses fptplay CDN as primary source (accessed via backend proxy with Referer header)
+Fetches stream URLs from hd.xemtv.net PHP endpoints for VTV1-VTV10 + VTV Cần Thơ
+The PHP endpoints return jwplayer config with fresh stream URLs (important for VTV6 which has expiring signatures)
 """
 import requests, re, time, threading
-from bs4 import BeautifulSoup
 
 UA = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
@@ -12,36 +11,53 @@ UA = {
     "Referer": "https://hd.xemtv.net/",
 }
 
-# Channel page URLs on xemtv.net (for scraping)
-XEMTV_CHANNELS = {
-    "VTV1": "https://hd.xemtv.net/kenh-vtv1-truc-tuyen-hot.html",
-    "VTV2": "https://hd.xemtv.net/kenh-vtv2-truc-tuyen-hot.html",
-    "VTV3": "https://hd.xemtv.net/kenh-vtv3-truc-tuyen-hot.html",
-    "VTV4": "https://hd.xemtv.net/kenh-vtv4-truc-tuyen-hot.html",
-    "VTV5": "https://hd.xemtv.net/kenh-vtv5-truc-tuyen-hot.html",
-    "VTV CẦN THƠ": "https://hd.xemtv.net/kenh-vtv10-truc-tuyen-hot.html",
-    "VTV7": "https://hd.xemtv.net/kenh-vtv7-truc-tuyen-hot.html",
-    "VTV8": "https://hd.xemtv.net/kenh-vtv8-truc-tuyen-hot.html",
-    "VTV9": "https://hd.xemtv.net/kenh-vtv9-truc-tuyen-hot.html",
+# Channel ID -> xemtv.net PHP endpoint mapping
+# These PHP pages return jwplayer config with fresh stream URLs
+XEMTV_PHP_ENDPOINTS = {
+    "vtv1":  "https://hd.xemtv.net/kenh/vtv1.php",
+    "vtv2":  "https://hd.xemtv.net/kenh/vtv2.php",
+    "vtv3":  "https://hd.xemtv.net/kenh/vtv3.php",
+    "vtv4":  "https://hd.xemtv.net/kenh/vtv4.php",
+    "vtv5":  "https://hd.xemtv.net/kenh/vtv5.php",
+    "vtv6":  "https://hd.xemtv.net/kenh/vtv6.php",
+    "vtv7":  "https://hd.xemtv.net/kenh/vtv7.php",
+    "vtv8":  "https://hd.xemtv.net/kenh/vtv8.php",
+    "vtv9":  "https://hd.xemtv.net/kenh/vtv9.php",
+    "vtv10": "https://hd.xemtv.net/kenh/vtv10.php",  # VTV Cần Thơ
 }
 
-# Primary CDN streams (fptplay) — these require Referer: https://fptplay.vn/
-# The backend proxy (/api/proxy/hls) adds this header
-CDN_STREAMS = {
-    "VTV1": "https://live.fptplay53.net/fnxch2/vtv1hd_abr.smil/chunklist.m3u8",
-    "VTV2": "https://live.fptplay53.net/fnxch2/vtv2hd_abr.smil/chunklist.m3u8",
-    "VTV3": "https://live.fptplay53.net/fnxch2/vtv3hd_abr.smil/chunklist.m3u8",
-    "VTV4": "https://live.fptplay53.net/fnxch2/vtv4hd_abr.smil/chunklist.m3u8",
-    "VTV5": "https://live-a.fptplay53.net/live/media/VTV5HD/live_hls_avc/index.m3u8",
-    "VTV CẦN THƠ": "https://live.fptplay53.net/fnxch2/vtvcantho_abr.smil/chunklist.m3u8",
-    "VTV7": "https://live.fptplay53.net/fnxhd1/vtv7hd_vhls.smil/chunklist_b5000000.m3u8",
-    "VTV8": "https://live.fptplay53.net/epzhd1/vtv8hd_vhls.smil/chunklist.m3u8",
-    "VTV9": "https://live.fptplay53.net/fnxhd1/vtv9hd_vhls.smil/chunklist.m3u8",
+# Channel display names
+CHANNEL_NAMES = {
+    "vtv1": "VTV1",
+    "vtv2": "VTV2",
+    "vtv3": "VTV3",
+    "vtv4": "VTV4",
+    "vtv5": "VTV5",
+    "vtv6": "VTV6",
+    "vtv7": "VTV7",
+    "vtv8": "VTV8",
+    "vtv9": "VTV9",
+    "vtv10": "VTV Cần Thơ",
+}
+
+# Fallback CDN streams (fptplay) — used when xemtv scraping fails
+CDN_FALLBACK = {
+    "vtv1":  "https://live.fptplay53.net/fnxch2/vtv1hd_abr.smil/chunklist.m3u8",
+    "vtv2":  "https://live.fptplay53.net/fnxch2/vtv2hd_abr.smil/chunklist.m3u8",
+    "vtv3":  "https://live.fptplay53.net/fnxch2/vtv3hd_abr.smil/chunklist.m3u8",
+    "vtv4":  "https://live.fptplay53.net/fnxch2/vtv4hd_abr.smil/chunklist.m3u8",
+    "vtv5":  "https://live-a.fptplay53.net/live/media/VTV5HD/live_hls_avc/index.m3u8",
+    "vtv7":  "https://live.fptplay53.net/fnxhd1/vtv7hd_vhls.smil/chunklist_b5000000.m3u8",
+    "vtv8":  "https://live.fptplay53.net/epzhd1/vtv8hd_vhls.smil/chunklist.m3u8",
+    "vtv9":  "https://live.fptplay53.net/fnxhd1/vtv9hd_vhls.smil/chunklist.m3u8",
+    "vtv10": "https://live.fptplay53.net/fnxch2/vtvcantho_abr.smil/chunklist.m3u8",
+    # VTV6 fallback: try canthotv as alternative
+    "vtv6":  "https://live.canthotv.vn/live/tv/chunklist.m3u8",
 }
 
 _vtv_cache = {}
 _vtv_lock = threading.Lock()
-_CACHE_TTL = 300  # 5 minutes
+_CACHE_TTL = 180  # 3 minutes — refresh frequently for VTV6 sign URLs
 
 
 def _cached(key):
@@ -57,84 +73,84 @@ def _set_cache(key, data):
 
 
 def extract_m3u8_from_html(html):
-    """Extract m3u8 URL from page HTML."""
+    """Extract m3u8 URL from xemtv PHP page (jwplayer config)."""
     if not html:
         return None
-    patterns = [
-        r'(https?://[^"\'<>\s]+\.m3u8[^"\'<>\s]*)',
-        r'file\s*[:=]\s*["\']([^"\']*\.m3u8[^"\']*)',
-        r'source\s*[:=]\s*["\']([^"\']*\.m3u8[^"\']*)',
-        r'src\s*[:=]\s*["\']([^"\']*\.m3u8[^"\']*)',
-        r'url\s*[:=]\s*["\']([^"\']*\.m3u8[^"\']*)',
-        r'hls\.loadSource\(["\']([^"\']*\.m3u8[^"\']*)',
-        r'player\.src\s*=\s*["\']([^"\']*\.m3u8[^"\']*)',
-        r'"file"\s*:\s*"([^"]*\.m3u8[^"]*)',
-        r'"src"\s*:\s*"([^"]*\.m3u8[^"]*)',
-        r'"url"\s*:\s*"([^"]*\.m3u8[^"]*)',
-    ]
-    for pat in patterns:
-        m = re.search(pat, html, re.IGNORECASE)
-        if m:
-            url = m.group(1).strip()
-            if '.m3u8' in url and len(url) > 20:
-                return url
+    # jwplayer config: file: 'URL'
+    m = re.search(r"file\s*:\s*['\"]([^'\"]*\.m3u8[^'\"]*)['\"]", html, re.IGNORECASE)
+    if m:
+        url = m.group(1).strip()
+        if len(url) > 20:
+            return url
+    # Generic m3u8 pattern
+    m = re.search(r"(https?://[^\s\"'<>\\]+\.m3u8[^\s\"'<>\\]*)", html, re.IGNORECASE)
+    if m:
+        url = m.group(1).strip()
+        if len(url) > 20:
+            return url
     return None
 
 
-def fetch_vtv_stream(channel_name):
-    """Fetch m3u8 stream URL for a VTV channel from xemtv.net."""
-    channel_name = channel_name.upper().strip()
+def fetch_vtv_stream(channel_id):
+    """Fetch m3u8 stream URL for a VTV channel by scraping xemtv.net PHP endpoint."""
+    channel_id = channel_id.lower().strip()
 
     # Normalize name
     name_map = {
-        'VTVCT': 'VTV CẦN THƠ', 'VTV-CAN-THO': 'VTV CẦN THƠ',
-        'VTV CAN THO': 'VTV CẦN THƠ', 'VTV6': 'VTV CẦN THƠ',
-        'VTCT': 'VTV CẦN THƠ',
+        'vtvct': 'vtv10', 'vtv-can-tho': 'vtv10', 'vtv can tho': 'vtv10',
+        'vtv_can_tho': 'vtv10', 'cantho': 'vtv10', 'cần thơ': 'vtv10',
+        'vietnam_vtv1': 'vtv1', 'vietnam_vtv2': 'vtv2', 'vietnam_vtv3': 'vtv3',
+        'vietnam_vtv4': 'vtv4', 'vietnam_vtv5': 'vtv5', 'vietnam_vtv6': 'vtv6',
+        'vietnam_vtv7': 'vtv7', 'vietnam_vtv8': 'vtv8', 'vietnam_vtv9': 'vtv9',
     }
-    channel_name = name_map.get(channel_name, channel_name)
+    channel_id = name_map.get(channel_id, channel_id)
 
     # Check cache
-    cached = _cached(channel_name)
+    cached = _cached(channel_id)
     if cached:
         return cached
 
-    page_url = XEMTV_CHANNELS.get(channel_name)
-    if not page_url:
-        cdn = CDN_STREAMS.get(channel_name)
-        if cdn:
-            _set_cache(channel_name, cdn)
-            return cdn
+    php_url = XEMTV_PHP_ENDPOINTS.get(channel_id)
+    if not php_url:
+        # Fallback
+        fallback = CDN_FALLBACK.get(channel_id)
+        if fallback:
+            _set_cache(channel_id, fallback)
+            return fallback
         return None
 
     try:
-        r = requests.get(page_url, headers=UA, timeout=15, allow_redirects=True)
+        r = requests.get(php_url, headers=UA, timeout=15, allow_redirects=True)
         if r.status_code == 200:
             m3u8 = extract_m3u8_from_html(r.text)
             if m3u8:
-                _set_cache(channel_name, m3u8)
+                _set_cache(channel_id, m3u8)
                 return m3u8
     except:
         pass
 
     # Fallback to CDN
-    cdn = CDN_STREAMS.get(channel_name)
-    if cdn:
-        _set_cache(channel_name, cdn)
-        return cdn
+    fallback = CDN_FALLBACK.get(channel_id)
+    if fallback:
+        _set_cache(channel_id, fallback)
+        return fallback
 
     return None
 
 
-def get_vtv_channels_for_frontend():
-    """Get all VTV channels with their stream URLs for the frontend."""
+def get_all_vtv_streams():
+    """Fetch all VTV channel streams. Returns list of {id, name, stream_url}."""
     channels = []
-    for name, page_url in XEMTV_CHANNELS.items():
-        stream_url = fetch_vtv_stream(name)
-        cdn_url = CDN_STREAMS.get(name, '')
+    for ch_id, php_url in XEMTV_PHP_ENDPOINTS.items():
+        stream_url = fetch_vtv_stream(ch_id)
         channels.append({
-            'name': name,
-            'page_url': page_url,
-            'stream_url': stream_url or cdn_url,
-            'cdn_fallback': cdn_url,
+            'id': ch_id,
+            'name': CHANNEL_NAMES.get(ch_id, ch_id.upper()),
+            'stream_url': stream_url,
         })
     return channels
+
+
+# Legacy compatibility
+XEMTV_CHANNELS = {v: k for k, v in CHANNEL_NAMES.items()}
+CDN_STREAMS = {v: k for k, v in CDN_FALLBACK.items()}