Upload vtv_scraper.py

vtv_scraper.py

@@ -0,0 +1,224 @@
+"""
+VTV Channel Scraper - Fetches live stream URLs from xemtv.net
+Scrapes VTV1-VTV10 + VTV Cần Thơ stream URLs dynamically.
+"""
+import requests, re, json, time, threading
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+
+UA = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+    "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+}
+REFERER = "https://hd.xemtv.net/"
+
+# Known xemtv.net channel page patterns
+XEMTV_CHANNELS = {
+    "VTV1": "https://hd.xemtv.net/kenh-vtv1-truc-tuyen-hot.html",
+    "VTV2": "https://hd.xemtv.net/kenh-vtv2-truc-tuyen-hot.html",
+    "VTV3": "https://hd.xemtv.net/kenh-vtv3-truc-tuyen-hot.html",
+    "VTV4": "https://hd.xemtv.net/kenh-vtv4-truc-tuyen-hot.html",
+    "VTV5": "https://hd.xemtv.net/kenh-vtv5-truc-tuyen-hot.html",
+    "VTV6": "https://hd.xemtv.net/kenh-vtv-can-tho-truc-tuyen-hot.html",  # VTV Cần Thơ
+    "VTV7": "https://hd.xemtv.net/kenh-vtv7-truc-tuyen-hot.html",
+    "VTV8": "https://hd.xemtv.net/kenh-vtv8-truc-tuyen-hot.html",
+    "VTV9": "https://hd.xemtv.net/kenh-vtv9-truc-tuyen-hot.html",
+    "VTV10": "https://hd.xemtv.net/kenh-vtv10-truc-tuyen-hot.html",
+}
+
+# Fallback: try alternate URL patterns
+XEMTV_ALT_PATTERNS = [
+    "https://hd.xemtv.net/kenh-{slug}-truc-tuyen-hot.html",
+    "https://hd.xemtv.net/{slug}-truc-tuyen-hot.html",
+    "https://xemtv.net/kenh-{slug}-truc-tuyen-hot.html",
+]
+
+_vtv_cache = {}
+_vtv_cache_lock = threading.Lock()
+_cache_ttl = 600  # 10 minutes
+
+
+def _cached(key):
+    with _vtv_cache_lock:
+        if key in _vtv_cache and time.time() - _vtv_cache[key]['t'] < _cache_ttl:
+            return _vtv_cache[key]['d']
+    return None
+
+
+def _set_cache(key, data):
+    with _vtv_cache_lock:
+        _vtv_cache[key] = {'t': time.time(), 'd': data}
+
+
+def _extract_stream_from_page(html, page_url=""):
+    """Extract m3u8/stream URL from xemtv.net page HTML."""
+    streams = []
+
+    # Pattern 1: direct m3u8 URLs in page source
+    for m in re.finditer(r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)', html):
+        url = m.group(1).strip()
+        if url not in streams:
+            streams.append(url)
+
+    # Pattern 2: m3u8 in JavaScript variables
+    for m in re.finditer(r'(?:src|source|url|file|stream|play_url|video_url)\s*[:=]\s*["\']([^"\']*\.m3u8[^"\']*)["\']', html, re.I):
+        url = m.group(1).strip()
+        if url not in streams:
+            streams.append(url)
+
+    # Pattern 3: in JSON-like config blocks
+    for m in re.finditer(r'"(?:src|source|url|file|stream|play_url|video_url|m3u8)"\s*:\s*"(https?://[^"]*\.m3u8[^"]*)"', html, re.I):
+        url = m.group(1).strip()
+        if url not in streams:
+            streams.append(url)
+
+    # Pattern 4: iframe src that might contain stream
+    for m in re.finditer(r'<iframe[^>]+src=["\']([^"\']+)["\']', html, re.I):
+        iframe_url = m.group(1).strip()
+        if 'xemtv' in iframe_url or 'embed' in iframe_url.lower():
+            streams.append(("iframe", iframe_url))
+
+    # Pattern 5: video/source tags
+    soup = BeautifulSoup(html, 'lxml')
+    for video in soup.find_all('video'):
+        src = video.get('src', '')
+        if src and '.m3u8' in src:
+            streams.append(src)
+        for source in video.find_all('source'):
+            src = source.get('src', '')
+            if src and '.m3u8' in src and src not in streams:
+                streams.append(src)
+
+    # Pattern 6: data-* attributes
+    for tag in soup.find_all(attrs={"data-url": True}):
+        url = tag['data-url']
+        if '.m3u8' in url and url not in streams:
+            streams.append(url)
+    for tag in soup.find_all(attrs={"data-src": True}):
+        url = tag['data-src']
+        if '.m3u8' in url and url not in streams:
+            streams.append(url)
+    for tag in soup.find_all(attrs={"data-stream": True}):
+        url = tag['data-stream']
+        if '.m3u8' in url and url not in streams:
+            streams.append(url)
+
+    # Filter: only keep direct m3u8 URLs (not iframe tuples)
+    direct = [s for s in streams if isinstance(s, str) and s.startswith('http')]
+    return direct
+
+
+def _extract_stream_via_jina(page_url):
+    """Use Jina AI to fetch page and extract stream URLs."""
+    try:
+        r = requests.get(
+            "https://r.jina.ai/" + page_url,
+            headers={"Accept": "text/markdown", "X-Return-Format": "markdown", "User-Agent": "Mozilla/5.0"},
+            timeout=30
+        )
+        if r.status_code == 200:
+            # Jina returns markdown, but we need the raw HTML for stream extraction
+            # Try to find m3u8 in the markdown output
+            for m in re.finditer(r'(https?://[^\s<>)]+\.m3u8[^\s<>)]*)', r.text):
+                return [m.group(1)]
+    except:
+        pass
+    return []
+
+
+def fetch_vtv_stream(channel_name):
+    """Fetch stream URL for a specific VTV channel from xemtv.net."""
+    cache_key = f"vtv_{channel_name}"
+    cached = _cached(cache_key)
+    if cached:
+        return cached
+
+    url = XEMTV_CHANNELS.get(channel_name)
+    if not url:
+        return None
+
+    streams = []
+
+    # Method 1: Direct fetch
+    try:
+        r = requests.get(url, headers={**UA, "Referer": REFERER}, timeout=15, allow_redirects=True)
+        if r.status_code == 200:
+            r.encoding = 'utf-8'
+            streams = _extract_stream_from_page(r.text, url)
+    except:
+        pass
+
+    # Method 2: Jina AI
+    if not streams:
+        streams = _extract_stream_via_jia(url)
+
+    # Method 3: Try alternate URL patterns
+    if not streams:
+        slug = channel_name.lower().replace(' ', '-')
+        for pattern in XEMTV_ALT_PATTERNS:
+            alt_url = pattern.format(slug=slug)
+            try:
+                r = requests.get(alt_url, headers={**UA, "Referer": REFERER}, timeout=10, allow_redirects=True)
+                if r.status_code == 200:
+                    r.encoding = 'utf-8'
+                    streams = _extract_stream_from_page(r.text, alt_url)
+                    if streams:
+                        break
+            except:
+                pass
+
+    result = streams[0] if streams else None
+    _set_cache(cache_key, result)
+    return result
+
+
+def fetch_all_vtv_streams():
+    """Fetch all VTV channel streams. Returns {channel_name: stream_url}."""
+    cache_key = "vtv_all"
+    cached = _cached(cache_key)
+    if cached:
+        return cached
+
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    results = {}
+
+    def _fetch_one(name):
+        url = fetch_vtv_stream(name)
+        return name, url
+
+    with ThreadPoolExecutor(5) as ex:
+        futs = [ex.submit(_fetch_one, name) for name in XEMTV_CHANNELS]
+        for f in as_completed(futs, timeout=30):
+            try:
+                name, url = f.result()
+                if url:
+                    results[name] = url
+            except:
+                pass
+
+    _set_cache(cache_key, results)
+    return results
+
+
+def get_vtv_channels_for_frontend():
+    """
+    Returns channel data for the frontend.
+    Tries xemtv.net first, falls back to known working streams.
+    """
+    streams = fetch_all_vtv_streams()
+
+    # Build ordered channel list
+    channels = []
+    for name in ["VTV1", "VTV2", "VTV3", "VTV4", "VTV5", "VTV6", "VTV7", "VTV8", "VTV9", "VTV10"]:
+        url = streams.get(name)
+        display_name = name if name != "VTV6" else "VTV Cần Thơ"
+        channels.append({
+            "name": name,
+            "displayName": display_name,
+            "url": url or "",
+            "logo": f"https://upload.wikimedia.org/wikipedia/commons/thumb/4/4f/{name}_logo.svg/200px-{name}_logo.svg.png"
+        })
+
+    return channels