Fix WC2026: scrape fixtures+standings from dantri.com.vn/the-thao/world-cup (server-rendered)"

wc2026_scraper.py

@@ -1,10 +1,10 @@
 """
 World Cup 2026 Data Module
-- Lịch thi đấu: static JSON (official FIFA schedule, 104 matches, giờ VN)
-- BXH: bongda.com.vn API tournament_id=24254
+- Lịch thi đấu + BXH: dantri.com.vn/the-thao/world-cup/ (server-rendered, scrapeable)
+- Stats: bongda.com.vn API tournament_id=24254
 - News: Thanh Niên + TT&VH + VnExpress
 """
-import requests, re, time, threading, json, os
+import requests, re, time, threading
 from bs4 import BeautifulSoup
 from urllib.parse import quote
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -16,7 +16,7 @@ BONGDA_HEADERS = {
     "Referer": "https://bongda.com.vn/giai-dau/24254/standings/world-cup",
     "X-Requested-With": "XMLHttpRequest"
 }
-UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'Accept-Language': 'vi-VN,vi;q=0.9'}
 WC_ID = 24254
 CACHE = {}
 LOCK = threading.Lock()
@@ -45,53 +45,75 @@ def _fetch(url, timeout=15):
         return r.text if r.status_code == 200 else ''
     except: return ''
 
-# ==================== LỊCH THI ĐẤU (static JSON - official FIFA data) ====================
+# ==================== LỊCH THI ĐẤU (dantri.com.vn) ====================
 def scrape_fixtures():
-    """Load official WC2026 fixtures from static JSON file."""
-    c = _cached('wc_fix', 86400)  # Cache 24h - data is static
+    """Scrape lịch thi đấu WC2026 từ dantri.com.vn/the-thao/world-cup/lich-thi-dau.htm"""
+    c = _cached('wc_fix', 300)
     if c is not None: return c
 
-    matches = []
-    json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static', 'wc2026_fixtures.json')
-    try:
-        with open(json_path, 'r', encoding='utf-8') as f:
-            matches = json.load(f)
-    except:
-        pass
-
-    # Group matches by group
-    groups = {}
-    for m in matches:
-        g = m.get('group', 'Khác')
-        if g not in groups: groups[g] = []
-        groups[g].append(m)
-
-    group_list = [{'group': k, 'matches': v} for k, v in groups.items()]
-    r = {'groups': group_list, 'total_matches': len(matches)}
+    html_content = ''
+    page = _fetch('https://dantri.com.vn/the-thao/world-cup/lich-thi-dau.htm')
+    if page:
+        soup = BeautifulSoup(page, 'lxml')
+        # Remove junk
+        for s in soup.select('script, style, nav, footer, header, .ads, .banner, .sidebar, .social-share, .comment-wrap, .relate-news'):
+            s.decompose()
+        # Dân Trí schedule page uses specific containers
+        content = (
+            soup.select_one('.schedule-container') or
+            soup.select_one('.match-schedule') or
+            soup.select_one('.livescore-container') or
+            soup.select_one('.content-schedule') or
+            soup.select_one('#schedule-content') or
+            soup.select_one('.singular-content') or
+            soup.select_one('main .e-magazine__body') or
+            soup.select_one('article') or
+            soup.select_one('main')
+        )
+        if content:
+            # Clean internal junk
+            for s in content.select('.ads, .banner, script, style, .social, .relate'):
+                s.decompose()
+            html_content = str(content)
+
+    r = {'html': html_content}
     _set('wc_fix', r)
     return r
 
-def debug_fixtures():
-    """Debug: show what data is loaded."""
-    json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static', 'wc2026_fixtures.json')
-    exists = os.path.exists(json_path)
-    count = 0
-    if exists:
-        try:
-            with open(json_path, 'r', encoding='utf-8') as f:
-                data = json.load(f)
-                count = len(data)
-        except: pass
-    return {'file_exists': exists, 'path': json_path, 'match_count': count}
-
-# ==================== BXH ====================
+# ==================== BXH (dantri.com.vn) ====================
 def scrape_standings():
-    c = _cached('wc_bxh', 180)
+    """Scrape BXH WC2026 từ dantri.com.vn/the-thao/world-cup/bang-xep-hang.htm"""
+    c = _cached('wc_bxh', 300)
     if c is not None: return c
-    html = _bongda(f"/api/league-table/home?tournament_id={WC_ID}&is_detail=True")
-    if not html:
-        html = _bongda(f"/api/league-table/home?tournament_id={WC_ID}")
-    r = {'html': html}
+
+    html_content = ''
+    # Try dantri first
+    page = _fetch('https://dantri.com.vn/the-thao/world-cup/bang-xep-hang.htm')
+    if page:
+        soup = BeautifulSoup(page, 'lxml')
+        for s in soup.select('script, style, nav, footer, header, .ads, .banner, .sidebar, .social-share, .comment-wrap, .relate-news'):
+            s.decompose()
+        content = (
+            soup.select_one('.standings-container') or
+            soup.select_one('.ranking-container') or
+            soup.select_one('.livescore-container') or
+            soup.select_one('#standings-content') or
+            soup.select_one('.singular-content') or
+            soup.select_one('article') or
+            soup.select_one('main')
+        )
+        if content:
+            for s in content.select('.ads, .banner, script, style, .social, .relate'):
+                s.decompose()
+            html_content = str(content)
+
+    # Fallback to bongda.com.vn API if dantri empty
+    if not html_content or len(html_content) < 100:
+        html_content = _bongda(f"/api/league-table/home?tournament_id={WC_ID}&is_detail=True")
+        if not html_content:
+            html_content = _bongda(f"/api/league-table/home?tournament_id={WC_ID}")
+
+    r = {'html': html_content}
     _set('wc_bxh', r)
     return r
 
@@ -105,17 +127,23 @@ def scrape_stats():
     return r
 
 # ==================== OTHER ====================
-def scrape_history(): return scrape_standings()
-def scrape_h2h(event_id): return {'html': _bongda(f"/api/fixtures/head-to-head?event_id={event_id}")}
-def scrape_lineups(event_id): return {'html': _bongda(f"/api/fixtures/lineups?event_id={event_id}")}
-def scrape_match_detail(event_id): return {'html': _bongda(f"/api/fixtures/commentaries?event_id={event_id}")}
-def scrape_summary(): return scrape_standings()
+def scrape_history():
+    return scrape_standings()
+def scrape_h2h(event_id):
+    return {'html': _bongda(f"/api/fixtures/head-to-head?event_id={event_id}")}
+def scrape_lineups(event_id):
+    return {'html': _bongda(f"/api/fixtures/lineups?event_id={event_id}")}
+def scrape_match_detail(event_id):
+    return {'html': _bongda(f"/api/fixtures/commentaries?event_id={event_id}")}
+def scrape_summary():
+    return scrape_standings()
 
 # ==================== NEWS ====================
 def scrape_wc_news():
     c = _cached('wc_news', 300)
     if c is not None: return c
     news = []
+    # Thanh Niên WC2026
     try:
         html = _fetch('https://worldcup2026.thanhnien.vn/')
         if html:
@@ -131,21 +159,35 @@ def scrape_wc_news():
                     news.append({'title': title, 'link': href, 'img': img, 'source': 'Thanh Niên'})
                 if len(news) >= 10: break
     except: pass
+    # Dân Trí WC
+    try:
+        html = _fetch('https://dantri.com.vn/the-thao/world-cup.htm')
+        if html:
+            soup = BeautifulSoup(html, 'lxml')
+            for a in soup.select('h3 a[href], .article-title a[href], .news-item a[href]')[:10]:
+                title = _clean(a.get('title', '') or a.get_text())
+                href = a.get('href', '')
+                if not href.startswith('http'): href = 'https://dantri.com.vn' + href
+                if title and len(title) > 15 and href not in [n['link'] for n in news]:
+                    news.append({'title': title, 'link': href, 'img': '', 'source': 'Dân Trí'})
+    except: pass
+    # TT&VH RSS
     try:
         html = _fetch('https://thethaovanhoa.vn/rss/world-cup-2026.rss')
         if html:
             soup = BeautifulSoup(html, 'xml')
-            for item in soup.find_all('item')[:10]:
+            for item in soup.find_all('item')[:8]:
                 title = _clean(item.find('title').get_text() if item.find('title') else '')
                 link = _clean(item.find('link').get_text() if item.find('link') else '')
                 if title and link and link not in [n['link'] for n in news]:
                     news.append({'title': title, 'link': link, 'img': '', 'source': 'TT&VH'})
     except: pass
+    # VnExpress
     try:
         html = _fetch(f'https://timkiem.vnexpress.net/?q={quote("World Cup 2026")}')
         if html:
             soup = BeautifulSoup(html, 'lxml')
-            for art in soup.select('article.item-news')[:8]:
+            for art in soup.select('article.item-news')[:6]:
                 a = art.select_one('h2 a, h3 a')
                 if not a: continue
                 title = _clean(a.get('title', '') or a.get_text())