Debug: add /api/wc2026/debug endpoint to check what tienphong.vn returns"

wc2026_scraper.py

@@ -1,8 +1,8 @@
 """
 World Cup 2026 Data Module
-- Lịch thi đấu: Parse từ tienphong.vn → trả JSON structured data → frontend render CSS layout giống ảnh
+- Lịch thi đấu: Parse từ tienphong.vn → JSON → frontend render CSS
 - BXH: bongda.com.vn API tournament_id=24254
-- News: VnExpress + TT&VH + Thanh Niên
+- News: Thanh Niên + TT&VH + VnExpress
 """
 import requests, re, time, threading
 from bs4 import BeautifulSoup
@@ -45,122 +45,223 @@ def _fetch(url, timeout=15):
         return r.text if r.status_code == 200 else ''
     except: return ''
 
-# ==================== LỊCH THI ĐẤU (parse structured JSON from tienphong.vn) ====================
+# ==================== LỊCH THI ĐẤU ====================
 def scrape_fixtures():
-    """Parse lịch thi đấu WC2026 từ tienphong.vn thành structured JSON."""
+    """Parse lịch thi đấu WC2026 từ tienphong.vn."""
     c = _cached('wc_fix', 3600)
     if c is not None: return c
 
-    groups = []  # [{group: "Bảng A", matches: [{date, time, home, away, venue, score}]}]
+    groups = []
+    debug_info = {'url': '', 'html_len': 0, 'tables_found': 0, 'text_matches': 0}
+    
+    # Try tienphong.vn
+    url = 'https://tienphong.vn/lich-thi-dau-world-cup-2026-moi-nhat-theo-gio-viet-nam-post1837627.tpo'
+    debug_info['url'] = url
+    page = _fetch(url)
+    debug_info['html_len'] = len(page) if page else 0
     
-    page = _fetch('https://tienphong.vn/lich-thi-dau-world-cup-2026-moi-nhat-theo-gio-viet-nam-post1837627.tpo')
     if page:
         soup = BeautifulSoup(page, 'lxml')
-        content = soup.select_one('.article__body, .cms-body, article, .content-detail, .article-body')
+        # Remove junk
+        for s in soup.select('script, style, nav, footer, .ads, .banner, .social-share, .related'):
+            s.decompose()
+        
+        # Find article content
+        content = (
+            soup.select_one('.article__body') or
+            soup.select_one('.cms-body') or 
+            soup.select_one('.article-content') or
+            soup.select_one('article') or
+            soup.select_one('.content-detail') or
+            soup.select_one('#main-detail') or
+            soup.select_one('.main-content')
+        )
+        
         if content:
-            # Parse tables in the article
-            current_group = ''
-            for el in content.find_all(['table', 'h2', 'h3', 'h4', 'p', 'strong', 'b']):
-                # Detect group/round headers
-                if el.name in ('h2', 'h3', 'h4', 'p', 'strong', 'b'):
+            # Method 1: Parse tables
+            tables = content.select('table')
+            debug_info['tables_found'] = len(tables)
+            
+            current_group = 'Vòng bảng'
+            for el in content.descendants:
+                if not hasattr(el, 'name') or not el.name: continue
+                
+                # Detect group headers from bold/strong/h tags
+                if el.name in ('h2', 'h3', 'h4', 'strong', 'b', 'p'):
                     text = _clean(el.get_text())
-                    if re.search(r'bảng\s+[A-L]|group\s+[A-L]|vòng\s+\d|vòng\s+bảng|vòng\s+1/16|tứ kết|bán kết|chung kết|knock.?out', text, re.I):
-                        current_group = text[:50]
-                elif el.name == 'table':
+                    if re.search(r'bảng\s+[A-L]|group\s+[A-L]|vòng\s+1/16|vòng\s+16|tứ kết|bán kết|chung kết|vòng\s+bảng|ngày\s+\d', text, re.I):
+                        if len(text) < 60:
+                            current_group = text
+                
+                # Parse table
+                if el.name == 'table':
                     rows = el.select('tr')
                     if len(rows) < 2: continue
                     group_matches = []
-                    for tr in rows[1:]:  # Skip header
+                    for tr in rows:
                         tds = tr.select('td')
-                        if len(tds) < 3: continue
+                        if len(tds) < 2: continue
                         cells = [_clean(td.get_text()) for td in tds]
-                        # Typical format: Ngày | Giờ | Đội 1 | Tỉ số | Đội 2 | Sân
-                        # Or: Trận | Ngày | Giờ | Đội 1 vs Đội 2 | Sân
-                        match = _parse_match_row(cells)
+                        match = _try_parse_cells(cells)
                         if match:
                             group_matches.append(match)
                     if group_matches:
-                        groups.append({'group': current_group or f'Vòng bảng', 'matches': group_matches})
-                        current_group = ''
-
-            # Fallback: parse from text patterns if no tables
+                        groups.append({'group': current_group, 'matches': group_matches})
+            
+            # Method 2: Parse text if no tables found
             if not groups:
-                text = content.get_text('\n')
-                current_group = ''
+                text_content = content.get_text('\n')
+                lines = text_content.split('\n')
+                current_group = 'Lịch thi đấu'
                 current_matches = []
-                for line in text.split('\n'):
+                
+                for line in lines:
                     line = _clean(line)
-                    if not line: continue
+                    if not line or len(line) < 5: continue
+                    
                     # Group header
-                    if re.search(r'bảng\s+[A-L]|group\s+[A-L]|vòng\s+\d|tứ kết|bán kết|chung kết', line, re.I):
-                        if current_matches and current_group:
+                    if re.search(r'bảng\s+[A-L]|group\s+[A-L]|vòng\s+1/16|tứ kết|bán kết|chung kết', line, re.I):
+                        if current_matches:
                             groups.append({'group': current_group, 'matches': current_matches})
                             current_matches = []
                         current_group = line[:50]
                         continue
-                    # Match pattern: "HH:MM ngày DD/MM: Team1 vs Team2 (Sân)"
-                    m = re.search(r'(\d{1,2}[h:]\d{2}).*?(\d{1,2}/\d{1,2}).*?[:\-–]\s*(.+?)\s+(?:vs|VS|–|-|v)\s+(.+?)(?:\s*\((.+?)\))?$', line)
-                    if m:
-                        current_matches.append({
-                            'time': m.group(1), 'date': m.group(2),
-                            'home': _clean(m.group(3)), 'away': _clean(m.group(4)),
-                            'venue': _clean(m.group(5)) if m.group(5) else '', 'score': ''
-                        })
-                        continue
-                    # Simpler pattern: "Team1 vs Team2"
-                    m2 = re.search(r'^(.{2,25})\s+(?:vs|VS|–)\s+(.{2,25})', line)
-                    if m2:
-                        tm = re.search(r'(\d{1,2}[h:]\d{2})', line)
-                        dt = re.search(r'(\d{1,2}/\d{1,2})', line)
-                        current_matches.append({
-                            'time': tm.group(1) if tm else '', 'date': dt.group(1) if dt else '',
-                            'home': _clean(m2.group(1)), 'away': _clean(m2.group(2)),
-                            'venue': '', 'score': ''
-                        })
+                    
+                    # Try parse match from line
+                    match = _try_parse_line(line)
+                    if match:
+                        current_matches.append(match)
+                        debug_info['text_matches'] += 1
+                
                 if current_matches:
-                    groups.append({'group': current_group or 'Lịch thi đấu', 'matches': current_matches})
-
-    r = {'groups': groups, 'total_matches': sum(len(g['matches']) for g in groups)}
+                    groups.append({'group': current_group, 'matches': current_matches})
+    
+    # Fallback: try thethao247.vn
+    if not groups:
+        groups = _try_thethao247()
+    
+    r = {'groups': groups, 'total_matches': sum(len(g['matches']) for g in groups), 'debug': debug_info}
     _set('wc_fix', r)
     return r
 
-def _parse_match_row(cells):
-    """Parse a table row into a match dict."""
-    if len(cells) < 3: return None
-    
+def _try_parse_cells(cells):
+    """Try to parse table cells into a match."""
+    if len(cells) < 2: return None
     date = ''; time_str = ''; home = ''; away = ''; venue = ''; score = ''
     
     for c in cells:
-        # Date pattern
+        if not c: continue
         if re.match(r'^\d{1,2}/\d{1,2}(/\d{2,4})?$', c) and not date:
             date = c; continue
-        # Time pattern
         if re.match(r'^\d{1,2}[h:]\d{2}$', c) and not time_str:
             time_str = c; continue
-        # Score pattern
         if re.match(r'^\d+\s*[-–]\s*\d+$', c):
             score = c; continue
-        # "vs" or dash separator
-        if c.lower() in ('vs', '-', '–', 'v'): continue
-        # Venue (usually longer, contains stadium words)
-        if any(w in c.lower() for w in ['stadium', 'sân', 'arena', 'park', 'field']) and not venue:
+        if c.lower() in ('vs', '-', '–', 'v', 'ft'): continue
+        # Check if cell contains "vs" pattern
+        vm = re.search(r'^(.+?)\s+(?:vs|VS|–|-)\s+(.+?)$', c)
+        if vm:
+            home = _clean(vm.group(1)); away = _clean(vm.group(2)); continue
+        if any(w in c.lower() for w in ['stadium', 'sân', 'arena', 'park', 'metlife', 'sofi', 'hard rock']):
             venue = c; continue
-        # Team names
-        if len(c) > 1 and not c.isdigit():
+        if len(c) > 2 and not c.isdigit() and len(c) < 30:
             if not home: home = c
             elif not away: away = c
     
-    # Handle "Team1 vs Team2" in single cell
-    if home and not away:
-        m = re.search(r'^(.+?)\s+(?:vs|VS|–|-|v)\s+(.+?)$', home)
-        if m:
-            home = _clean(m.group(1))
-            away = _clean(m.group(2))
-    
     if home and away:
         return {'date': date, 'time': time_str, 'home': home, 'away': away, 'venue': venue, 'score': score}
     return None
 
+def _try_parse_line(line):
+    """Try to parse a text line into a match."""
+    # Pattern: "21h00 ngày 12/6: Mexico vs Honduras (SoFi Stadium)"
+    m = re.search(r'(\d{1,2}[h:]\d{2})\s*(?:ngày\s*)?(\d{1,2}/\d{1,2}(?:/\d{2,4})?)?[:\s\-–]*(.+?)\s+(?:vs|VS|–|-|v\.?s\.?)\s+(.+?)(?:\s*[\(\[](.+?)[\)\]])?$', line)
+    if m:
+        return {
+            'time': _clean(m.group(1)), 'date': _clean(m.group(2) or ''),
+            'home': _clean(m.group(3)), 'away': _clean(m.group(4)),
+            'venue': _clean(m.group(5) or ''), 'score': ''
+        }
+    # Pattern: "Mexico - Honduras 21h 12/6"
+    m2 = re.search(r'^([A-ZÀ-Ỹ][a-zà-ỹA-ZÀ-Ỹ\s\.]+?)\s+(?:vs|VS|–|-)\s+([A-ZÀ-Ỹ][a-zà-ỹA-ZÀ-Ỹ\s\.]+)', line)
+    if m2:
+        tm = re.search(r'(\d{1,2}[h:]\d{2})', line)
+        dt = re.search(r'(\d{1,2}/\d{1,2})', line)
+        return {
+            'time': tm.group(1) if tm else '', 'date': dt.group(1) if dt else '',
+            'home': _clean(m2.group(1)), 'away': _clean(m2.group(2)),
+            'venue': '', 'score': ''
+        }
+    return None
+
+def _try_thethao247():
+    """Fallback: parse from thethao247.vn"""
+    groups = []
+    page = _fetch('https://thethao247.vn/world-cup/426-lich-thi-dau-world-cup-2026-d399919.html')
+    if not page: return groups
+    soup = BeautifulSoup(page, 'lxml')
+    content = soup.select_one('article, .detail-content, .content-detail, main')
+    if not content: return groups
+    
+    current_group = 'Lịch thi đấu'
+    current_matches = []
+    for el in content.find_all(['table', 'h2', 'h3', 'h4', 'strong']):
+        if el.name in ('h2', 'h3', 'h4', 'strong'):
+            text = _clean(el.get_text())
+            if re.search(r'bảng|group|vòng|tứ kết|bán kết|chung kết', text, re.I) and len(text) < 60:
+                if current_matches:
+                    groups.append({'group': current_group, 'matches': current_matches})
+                    current_matches = []
+                current_group = text
+        elif el.name == 'table':
+            for tr in el.select('tr')[1:]:
+                tds = tr.select('td')
+                if len(tds) >= 2:
+                    cells = [_clean(td.get_text()) for td in tds]
+                    match = _try_parse_cells(cells)
+                    if match: current_matches.append(match)
+    if current_matches:
+        groups.append({'group': current_group, 'matches': current_matches})
+    return groups
+
+def debug_fixtures():
+    """Debug endpoint to see raw data."""
+    page = _fetch('https://tienphong.vn/lich-thi-dau-world-cup-2026-moi-nhat-theo-gio-viet-nam-post1837627.tpo')
+    if not page:
+        return {'error': 'Cannot fetch tienphong.vn', 'page_len': 0}
+    
+    soup = BeautifulSoup(page, 'lxml')
+    for s in soup.select('script, style'): s.decompose()
+    
+    content = (
+        soup.select_one('.article__body') or soup.select_one('.cms-body') or
+        soup.select_one('article') or soup.select_one('.content-detail')
+    )
+    
+    info = {
+        'page_len': len(page),
+        'content_found': bool(content),
+        'content_selector': '',
+        'tables': 0,
+        'images': 0,
+        'text_preview': '',
+        'first_500_chars': ''
+    }
+    
+    if content:
+        info['content_selector'] = content.name + '.' + '.'.join(content.get('class', []))
+        info['tables'] = len(content.select('table'))
+        info['images'] = len(content.select('img'))
+        info['text_preview'] = content.get_text()[:2000]
+        info['first_500_chars'] = str(content)[:500]
+    else:
+        # Try body
+        body = soup.find('body')
+        if body:
+            info['text_preview'] = body.get_text()[:2000]
+    
+    return info
+
 # ==================== BXH ====================
 def scrape_standings():
     c = _cached('wc_bxh', 180)