FIX: scrape WC2026 fixtures from bongdanet.net (server-rendered, dedicated WC page)"

wc2026_scraper.py

@@ -1,8 +1,7 @@
 """
 World Cup 2026 Data Module
-- Lịch thi đấu: bongda24h.vn/vck-world-cup/lich-thi-dau-41.html (server-rendered, scrapeable)
-- BXH: bongda.com.vn API /api/league-table/home?tournament_id=24254 (PROVEN)
-- Stats: bongda.com.vn API
+- BXH: bongda.com.vn API (tournament_id=24254) - PROVEN WORKING
+- Fixtures: bongdanet.net/lich-thi-dau/world-cup-2026/ (server-rendered)
 - News: VnExpress + TT&VH + BongDaPlus
 """
 import requests, re, time, threading
@@ -17,12 +16,29 @@ BONGDA_HEADERS = {
     "Referer": "https://bongda.com.vn/giai-dau/24254/standings/world-cup",
     "X-Requested-With": "XMLHttpRequest"
 }
-UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'}
+UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'Accept-Language': 'vi-VN,vi;q=0.9'}
 WC_ID = 24254
 CACHE = {}
 LOCK = threading.Lock()
 _FIXTURES_ENDPOINT = None
-FIXTURES_CANDIDATES = []
+FIXTURES_CANDIDATES = [
+    f"/api/fixtures/home?tournament_id={WC_ID}",
+    f"/api/fixtures/home?tournament_id={WC_ID}&is_detail=True",
+    f"/api/fixtures/list?tournament_id={WC_ID}",
+    f"/api/fixtures/get-by-tournament?tournament_id={WC_ID}",
+    f"/api/fixtures/get-by-season?tournament_id={WC_ID}",
+    f"/api/fixtures/incoming?tournament_id={WC_ID}",
+    f"/api/fixtures/index?tournament_id={WC_ID}",
+    f"/api/tournament/{WC_ID}/fixtures",
+    f"/api/tournament-fixtures?tournament_id={WC_ID}",
+    f"/api/fixtures?tournament_id={WC_ID}",
+    f"/api/fixtures/all?tournament_id={WC_ID}",
+    f"/api/fixtures/schedule?tournament_id={WC_ID}",
+    f"/api/season-fixtures?tournament_id={WC_ID}",
+    f"/api/giai-dau/{WC_ID}/fixtures",
+    f"/api/fixtures/filter?tournament_id={WC_ID}",
+    f"/api/fixtures/by-tournament?tournament_id={WC_ID}",
+]
 
 def _clean(s): return re.sub(r'\s+', ' ', str(s or '')).strip()
 def _cached(key, ttl=120):
@@ -50,71 +66,73 @@ def _fetch(url, timeout=15):
         return r.text if r.status_code == 200 else ''
     except: return ''
 
-# ==================== FIXTURES from bongda24h.vn ====================
+# ==================== FIXTURES ====================
 def scrape_fixtures():
-    """Lịch thi đấu WC2026 from bongda24h.vn (server-rendered HTML)."""
+    """Lịch thi đấu WC2026 from bongdanet.net (server-rendered, reliable)."""
     c = _cached('wc_fix', 300)
     if c is not None: return c
-
+    
     html_out = ''
+    
+    # Source 1: bongdanet.net - dedicated WC2026 schedule page
     try:
-        page = _fetch('https://bongda24h.vn/vck-world-cup/lich-thi-dau-41.html')
-        if page:
+        page = _fetch('https://bongdanet.net/lich-thi-dau/world-cup-2026/')
+        if page and len(page) > 1000:
             soup = BeautifulSoup(page, 'lxml')
-            # Remove unwanted elements
-            for s in soup.select('script, style, .ads, .banner, .adv, iframe, .social-share, .breadcrumb, nav, header, footer, .sidebar'):
+            # Remove scripts, ads, nav
+            for s in soup.select('script, style, .ads, .banner, .adv, iframe, nav, header, footer, .sidebar, .breadcrumb'):
                 s.decompose()
-
-            # bongda24h.vn schedule page has match tables/divs
-            # Look for schedule content container
-            content = (
-                soup.select_one('.schedule-content') or
-                soup.select_one('.match-schedule') or
-                soup.select_one('.ltdContent') or
-                soup.select_one('.content-tab') or
-                soup.select_one('.box-content') or
-                soup.select_one('#schedule') or
-                soup.select_one('.main-content article') or
-                soup.select_one('.main-content')
-            )
-
-            if content:
-                # Remove nav, sidebars inside content
-                for s in content.select('.sidebar, .ads, .banner, .social, .breadcrumb, .related-news'):
-                    s.decompose()
+            # Find schedule content
+            content = soup.select_one('.schedule-content, .match-schedule, .main-content, .content, article, .post-content')
+            if content and len(content.get_text()) > 100:
                 html_out = str(content)
             else:
-                # Fallback: get all tables that look like schedules
-                tables = soup.select('table')
-                for t in tables:
-                    rows = t.select('tr')
-                    if len(rows) > 3:
-                        text = t.get_text().lower()
-                        if any(kw in text for kw in ['vs', ' - ', 'bảng', 'vòng', 'group']):
-                            html_out = str(t)
-                            break
-
-                # If still no table, try to get match divs
-                if not html_out:
-                    matches = soup.select('.match-item, .match-row, .item-schedule, .match-fixture, .event-row')
-                    if matches:
-                        html_out = '<div class="wc-matches">' + ''.join(str(m) for m in matches[:60]) + '</div>'
-
-                # Last resort: get body main content
+                # Try finding tables with match data
+                for table in soup.select('table'):
+                    if len(table.select('tr')) > 3:
+                        html_out = str(table)
+                        break
+                # Or divs with schedule
                 if not html_out:
-                    main = soup.select_one('main') or soup.select_one('.main-content') or soup.select_one('#content')
-                    if main:
-                        for s in main.select('.sidebar, .ads, nav, header, footer'): s.decompose()
-                        # Only keep elements with match-like content
-                        parts = []
-                        for el in main.select('table, .match-item, .schedule-round, .round-matches, div[class*=match], div[class*=schedule]'):
-                            parts.append(str(el))
-                        if parts:
-                            html_out = '<div>' + ''.join(parts[:30]) + '</div>'
-                        else:
-                            html_out = str(main)[:40000]
+                    for div in soup.select('div[class*=schedule], div[class*=match], div[class*=fixture]'):
+                        if len(div.get_text()) > 100:
+                            html_out = str(div)
+                            break
     except: pass
-
+    
+    # Source 2: webthethao.vn
+    if not html_out:
+        try:
+            page = _fetch('https://webthethao.vn/bong-da-quoc-te/lich-thi-dau-day-du-104-tran-dau-tai-world-cup-2026-theo-gio-viet-nam-WmooXjWvR.htm')
+            if page and len(page) > 1000:
+                soup = BeautifulSoup(page, 'lxml')
+                for s in soup.select('script, style, .ads, iframe, nav, header, footer'):
+                    s.decompose()
+                content = soup.select_one('.detail-content, .content-detail, article, .entry-content, .singular-content')
+                if content and len(content.get_text()) > 200:
+                    html_out = str(content)
+        except: pass
+    
+    # Source 3: 24h.com.vn
+    if not html_out:
+        try:
+            page = _fetch('https://www.24h.com.vn/bong-da/lich-thi-dau-bong-da-world-cup-2026-moi-nhat-c48a1747402.html')
+            if page and len(page) > 1000:
+                soup = BeautifulSoup(page, 'lxml')
+                for s in soup.select('script, style, .ads, iframe'):
+                    s.decompose()
+                # Look for schedule tables
+                for table in soup.select('table'):
+                    text = table.get_text().lower()
+                    if any(kw in text for kw in ['bảng', 'group', 'vs', ' - ', 'world cup']):
+                        html_out = str(table)
+                        break
+        except: pass
+    
+    # Truncate if too long
+    if html_out and len(html_out) > 50000:
+        html_out = html_out[:50000]
+    
     r = {'html': html_out}
     _set('wc_fix', r)
     return r
@@ -147,18 +165,14 @@ def scrape_history():
 
 def scrape_h2h(event_id):
     return {'html': _bongda(f"/api/fixtures/head-to-head?event_id={event_id}")}
-
 def scrape_lineups(event_id):
     return {'html': _bongda(f"/api/fixtures/lineups?event_id={event_id}")}
-
 def scrape_match_detail(event_id):
     return {'html': _bongda(f"/api/fixtures/commentaries?event_id={event_id}")}
-
 def scrape_summary():
     c = _cached('wc_sum', 90)
     if c is not None: return c
-    html = _bongda(f"/api/league-table/home?tournament_id={WC_ID}&is_detail=True")
-    r = {'html': html}
+    r = {'html': _bongda(f"/api/league-table/home?tournament_id={WC_ID}&is_detail=True")}
     _set('wc_sum', r)
     return r
 
@@ -245,7 +259,7 @@ def get_wc2026_all():
             ex.submit(scrape_wc_news): 'news',
             ex.submit(scrape_road_to_wc): 'road',
         }
-        for f in as_completed(futs, timeout=20):
+        for f in as_completed(futs, timeout=25):
             key = futs[f]
             try: data[key] = f.result()
             except: data[key] = {} if key in ('fixtures', 'standings', 'stats') else []