Upload match_detail_v2.py

match_detail_v2.py

@@ -1,7 +1,4 @@
-"""VNEWS — Match Detail Parser v2
-Parses bongda.com.vn match detail pages with correct selectors.
-Extracts timeline events (goals, cards, substitutions) and statistics.
-"""
+"""VNEWS — Match Detail Parser v2 (html.parser only, no lxml dependency)"""
 import re
 import requests
 from bs4 import BeautifulSoup
@@ -14,53 +11,39 @@ HEADERS = {
 }
 
 API_HEADERS = {
-    **HEADERS,
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
     "Accept": "application/json, text/javascript, */*; q=0.01",
     "X-Requested-With": "XMLHttpRequest",
+    "Referer": "https://bongda.com.vn/",
 }
 
 
-def _mk(html):
-    try:
-        return BeautifulSoup(html, 'lxml')
-    except Exception:
-        return BeautifulSoup(html, 'html.parser')
-
-
 def _cl(s):
     return re.sub(r'\s+', ' ', str(s or '')).strip()
 
 
-def fetch_html(url, timeout=15):
-    """Fetch HTML from URL."""
-    resp = requests.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
-    resp.raise_for_status()
-    return resp.text
-
-
 def _normalize_time(raw):
-    """Normalize time display: '45' +2' → '45+2'', '46'' → '46''', etc."""
     t = _cl(raw)
     if not t:
         return t
-    # "45' +2" → "45+2'"
     t = re.sub(r"(\d+)'\s*\+(\d+)", r"\1+\2'", t)
-    # "46''" → "46'" (double apostrophe fix)
     t = t.replace("''", "'")
     return t
 
 
+def _mk(html):
+    """Parse HTML using html.parser (lxml may not be available)."""
+    return BeautifulSoup(html, 'html.parser')
+
+
+def fetch_html(url, timeout=15):
+    resp = requests.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
+    resp.raise_for_status()
+    return resp.text
+
+
 def parse_events(sp):
-    """Parse timeline events from BeautifulSoup object.
-    
-    Structure: .events > .period > .event.home-team|away-team
-    
-    Each event has:
-      - .event-type with icon (goal/redcard/yellowcard/substitution SVG)
-      - .players with .event-time and player names inside .players.card|.goal|.subst
-    
-    Returns list of event dicts with: type, time, team, player_in, player_out, scorer, assist, player
-    """
+    """Parse .events > .period > .event structure."""
     events = []
     events_div = sp.select_one('.events')
     if not events_div:
@@ -70,7 +53,7 @@ def parse_events(sp):
     for child in events_div.children:
         if not hasattr(child, 'name') or not child.name:
             continue
-        cls_str = ' '.join(child.get('class', []))
+        cls_str = ' '.join(child.get('class', []) if child.get('class') else [])
 
         if 'period' in cls_str:
             h2 = child.find('h2')
@@ -80,76 +63,51 @@ def parse_events(sp):
             for ev in child.children:
                 if not hasattr(ev, 'name') or not ev.name:
                     continue
-                ev_cls_str = ' '.join(ev.get('class', []))
+                ev_cls_str = ' '.join(ev.get('class', []) if ev.get('class') else [])
                 if 'event' not in ev_cls_str:
                     continue
 
                 team = 'home' if 'home' in ev_cls_str else 'away'
                 ev_data = {
-                    'team': team,
-                    'period': current_period,
-                    'type': 'unknown',
-                    'time': '',
-                    'players': '',
-                    'player_in': '',
-                    'player_out': '',
-                    'scorer': '',
-                    'assist': '',
-                    'card_type': '',
-                    'player': '',
+                    'team': team, 'period': current_period, 'type': 'unknown',
+                    'time': '', 'players': '', 'player_in': '', 'player_out': '',
+                    'scorer': '', 'assist': '', 'card_type': '', 'player': '',
                 }
 
-                # Determine event type from icon in .event-type
                 type_el = ev.select_one('.event-type')
                 if type_el:
                     if type_el.select_one('[class*="redcard"]'):
-                        ev_data['type'] = 'redcard'
-                        ev_data['card_type'] = 'red'
+                        ev_data['type'] = 'redcard'; ev_data['card_type'] = 'red'
                     elif type_el.select_one('[class*="yellowcard"]'):
-                        ev_data['type'] = 'yellowcard'
-                        ev_data['card_type'] = 'yellow'
+                        ev_data['type'] = 'yellowcard'; ev_data['card_type'] = 'yellow'
                     elif type_el.select_one('[class*="goal"]'):
                         ev_data['type'] = 'goal'
                     elif type_el.select_one('[class*="substitution"]'):
                         ev_data['type'] = 'substitution'
                     else:
-                        # Check SVG rect color for red card
                         for rect in type_el.select('svg rect'):
                             if rect.get('fill') == '#E20007':
-                                ev_data['type'] = 'redcard'
-                                ev_data['card_type'] = 'red'
-                                break
-                        # Check SVG circle for goal (white filled circle = ball)
+                                ev_data['type'] = 'redcard'; ev_data['card_type'] = 'red'; break
                         if ev_data['type'] == 'unknown':
                             for circle in type_el.select('svg circle'):
                                 if circle.get('fill') == 'white' and circle.get('r') == '8':
-                                    ev_data['type'] = 'goal'
-                                    break
-                        # Fallback: check .players class for subst
+                                    ev_data['type'] = 'goal'; break
                         if ev_data['type'] == 'unknown' and ev.select_one('.players.subst'):
                             ev_data['type'] = 'substitution'
 
-                # Additional type detection from .players class
                 players_el = ev.select_one('.players')
                 if players_el and ev_data['type'] == 'unknown':
-                    pcls = ' '.join(players_el.get('class', []))
-                    if 'goal' in pcls:
-                        ev_data['type'] = 'goal'
-                    elif 'card' in pcls:
-                        ev_data['type'] = 'redcard'
-                        ev_data['card_type'] = 'red'
-                    elif 'subst' in pcls:
-                        ev_data['type'] = 'substitution'
+                    pcls = ' '.join(players_el.get('class', []) if players_el.get('class') else [])
+                    if 'goal' in pcls: ev_data['type'] = 'goal'
+                    elif 'card' in pcls: ev_data['type'] = 'redcard'; ev_data['card_type'] = 'red'
+                    elif 'subst' in pcls: ev_data['type'] = 'substitution'
 
-                # Extract time and player info
                 if players_el:
                     time_el = players_el.select_one('.event-time')
                     if time_el:
                         ev_data['time'] = _normalize_time(time_el.get_text())
-
                     ev_data['players'] = _cl(players_el.get_text(' ', strip=True))
 
-                    # Parse individual text nodes (excluding time)
                     texts = []
                     for d in players_el.find_all('div', recursive=False):
                         t = _cl(d.get_text())
@@ -162,50 +120,20 @@ def parse_events(sp):
 
                     if ev_data['type'] == 'substitution':
                         if len(texts) >= 2:
-                            ev_data['player_out'] = texts[0]
-                            ev_data['player_in'] = texts[1]
+                            ev_data['player_out'] = texts[0]; ev_data['player_in'] = texts[1]
                         elif len(texts) == 1:
                             ev_data['player_in'] = texts[0]
-                        # Fallback: parse from combined text like "46' Jose Sa Rui Silva"
-                        if not ev_data['player_in'] and ev_data['players']:
-                            remaining = ev_data['players'].replace(ev_data['time'], '').strip()
-                            words = remaining.split()
-                            if len(words) >= 4:
-                                mid = len(words) // 2
-                                ev_data['player_out'] = ' '.join(words[:mid])
-                                ev_data['player_in'] = ' '.join(words[mid:])
-
                     elif ev_data['type'] == 'goal':
-                        if len(texts) >= 1:
-                            ev_data['scorer'] = texts[0]
-                        if len(texts) >= 2:
-                            ev_data['assist'] = texts[1]
-                        # Fallback
-                        if not ev_data['scorer'] and ev_data['players']:
-                            remaining = ev_data['players'].replace(ev_data['time'], '').strip()
-                            words = remaining.split()
-                            if len(words) >= 2:
-                                ev_data['scorer'] = ' '.join(words[:2])
-                            elif len(words) == 1:
-                                ev_data['scorer'] = words[0]
-
+                        if len(texts) >= 1: ev_data['scorer'] = texts[0]
+                        if len(texts) >= 2: ev_data['assist'] = texts[1]
                     elif ev_data['type'] in ('redcard', 'yellowcard'):
-                        if texts:
-                            ev_data['player'] = ' '.join(texts)
-                        elif ev_data['players']:
-                            remaining = ev_data['players'].replace(ev_data['time'], '').strip()
-                            if remaining:
-                                ev_data['player'] = remaining
+                        if texts: ev_data['player'] = ' '.join(texts)
 
                 events.append(ev_data)
-
     return events
 
 
 def fetch_match_detail(event_id: int) -> dict:
-    """Fetch and parse match detail by event ID.
-    Tries /centre/, /preview/, /bao-cao-nhanh/ URL formats.
-    """
     result = {"event_id": event_id, "found": False, "sections": []}
 
     html = None
@@ -213,11 +141,11 @@ def fetch_match_detail(event_id: int) -> dict:
     for suffix in ['/centre/', '/preview/', '/bao-cao-nhanh/']:
         url = base + suffix
         try:
-            html = fetch_html(url, timeout=15)
-            if html and len(html) > 1000 and '.teams' in html:
+            resp = requests.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
+            if resp.status_code == 200 and len(resp.text) > 1000:
+                html = resp.text
                 break
         except Exception:
-            html = None
             continue
 
     if not html:
@@ -226,35 +154,28 @@ def fetch_match_detail(event_id: int) -> dict:
     sp = _mk(html)
     info = {}
 
-    # === Parse teams + score ===
     tel = sp.select_one('.teams')
     if tel:
         he = tel.select_one('.team.home') or tel.select_one('[class*="home"]')
         if he:
             ne = he.select_one('p:not(.logo)') or he.find('p')
-            if ne:
-                info['home_team'] = _cl(ne.get_text())
+            if ne: info['home_team'] = _cl(ne.get_text())
             lo = he.select_one('img')
-            if lo:
-                info['home_logo'] = lo.get('src', '')
+            if lo: info['home_logo'] = lo.get('src', '')
 
         ae = tel.select_one('.team.away') or tel.select_one('[class*="away"]')
         if ae:
             ne = ae.select_one('p:not(.logo)') or ae.find('p')
-            if ne:
-                info['away_team'] = _cl(ne.get_text())
+            if ne: info['away_team'] = _cl(ne.get_text())
             lo = ae.select_one('img')
-            if lo:
-                info['away_logo'] = lo.get('src', '')
+            if lo: info['away_logo'] = lo.get('src', '')
 
         sc = tel.select_one('.score')
         if sc:
             parts = [_cl(p.get_text()) for p in sc.select('p')]
-            if len(parts) >= 2:
-                info['score'] = f"{parts[0]} - {parts[1]}"
+            if len(parts) >= 2: info['score'] = f"{parts[0]} - {parts[1]}"
             lb = sc.select_one('.label')
-            if lb:
-                info['status_label'] = _cl(lb.get_text())
+            if lb: info['status_label'] = _cl(lb.get_text())
 
     if info.get('home_team') and info.get('away_team'):
         result['info'] = info
@@ -263,24 +184,19 @@ def fetch_match_detail(event_id: int) -> dict:
     else:
         return result
 
-    # === Parse match info (date/time) ===
     mi = sp.select_one('.match-info')
     if mi:
         for sel in ['.times', 'li']:
             el = mi.select_one(sel)
             if el:
                 t = _cl(el.get_text())
-                if t:
-                    info.setdefault('datetime', t)
-                    break
+                if t: info.setdefault('datetime', t); break
 
-    # === Parse events (timeline) ===
     events = parse_events(sp)
     if events:
         result['events'] = events
         result['sections'].append('events')
 
-    # === Parse prediction card ===
     pred = sp.select_one('.prediction-card')
     if pred:
         pred_data = {}
@@ -291,19 +207,15 @@ def fetch_match_detail(event_id: int) -> dict:
                 pred_data['home_name'] = _cl(teams[0].select_one('.team-name').get_text()) if teams[0].select_one('.team-name') else ''
                 pred_data['away_name'] = _cl(teams[1].select_one('.team-name').get_text()) if teams[1].select_one('.team-name') else ''
                 divider = team_info.select_one('.divider')
-                if divider:
-                    pred_data['result'] = _cl(divider.get_text())
+                if divider: pred_data['result'] = _cl(divider.get_text())
         vote_count = pred.select_one('.vote-count')
-        if vote_count:
-            pred_data['vote_count'] = _cl(vote_count.get_text())
+        if vote_count: pred_data['vote_count'] = _cl(vote_count.get_text())
         result['prediction'] = pred_data
 
-    # === Fetch H2H stats from API ===
     try:
         ar = requests.get(
             f"https://bongda.com.vn/api/fixtures/h2h-stats?event_id={event_id}",
-            headers=API_HEADERS,
-            timeout=10
+            headers=API_HEADERS, timeout=10
         )
         if ar.status_code == 200:
             ad = ar.json()
@@ -314,18 +226,11 @@ def fetch_match_detail(event_id: int) -> dict:
                     cells = row.select('td, span, p')
                     if len(cells) >= 3:
                         lb = _cl(cells[0].get_text())
-                        if lb:
-                            ast[lb] = {
-                                'home': _cl(cells[1].get_text()),
-                                'away': _cl(cells[2].get_text()),
-                            }
-                if ast:
-                    result['h2h_stats_parsed'] = ast
-                    result['sections'].append('h2h_stats')
+                        if lb: ast[lb] = {'home': _cl(cells[1].get_text()), 'away': _cl(cells[2].get_text())}
+                if ast: result['h2h_stats_parsed'] = ast; result['sections'].append('h2h_stats')
     except Exception:
         pass
 
-    # === Parse H2H standings table ===
     h2h_data = []
     h2h_el = sp.select_one('.h2h-standings')
     if h2h_el:
@@ -347,11 +252,8 @@ def fetch_match_detail(event_id: int) -> dict:
                     'ga': _cl(cells[6].get_text()) if len(cells) > 6 else '',
                     'points': _cl(cells[8].get_text()) if len(cells) > 8 else '',
                 })
-        if h2h_data:
-            result['h2h_standings'] = h2h_data
-            result['sections'].append('h2h_standings')
+        if h2h_data: result['h2h_standings'] = h2h_data; result['sections'].append('h2h_standings')
 
-    # === Parse recent matches from sidebar ===
     recent_matches = []
     matches_list = sp.select_one('.matches-list')
     if matches_list:
@@ -369,29 +271,27 @@ def fetch_match_detail(event_id: int) -> dict:
                     'away': _cl(away_el.get_text()) if away_el else '',
                     'score': _cl(score_el.get_text()) if score_el else 'vs',
                 })
-        if recent_matches:
-            result['recent_matches'] = recent_matches
-            result['sections'].append('recent')
+        if recent_matches: result['recent_matches'] = recent_matches; result['sections'].append('recent')
 
     return result
 
 
 def fetch_match_detail_by_url(url: str) -> dict:
-    """Fetch and parse match detail from a full bongda.com.vn URL (with slug)."""
     eid_match = re.search(r'/tran-dau/(\d+)/', url)
     if not eid_match:
         return {"event_id": 0, "found": False, "error": "Cannot extract event_id from URL"}
-
     event_id = int(eid_match.group(1))
     result = {"event_id": event_id, "found": False, "sections": []}
 
     html = None
     try:
-        html = fetch_html(url, timeout=15)
+        resp = requests.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
+        if resp.status_code == 200 and len(resp.text) > 1000:
+            html = resp.text
     except Exception:
         pass
 
-    if not html or len(html) < 1000 or '.teams' not in html:
+    if not html:
         return fetch_match_detail(event_id)
 
     sp = _mk(html)
@@ -402,47 +302,34 @@ def fetch_match_detail_by_url(url: str) -> dict:
         he = tel.select_one('.team.home') or tel.select_one('[class*="home"]')
         if he:
             ne = he.select_one('p:not(.logo)') or he.find('p')
-            if ne:
-                info['home_team'] = _cl(ne.get_text())
+            if ne: info['home_team'] = _cl(ne.get_text())
             lo = he.select_one('img')
-            if lo:
-                info['home_logo'] = lo.get('src', '')
-
+            if lo: info['home_logo'] = lo.get('src', '')
         ae = tel.select_one('.team.away') or tel.select_one('[class*="away"]')
         if ae:
             ne = ae.select_one('p:not(.logo)') or ae.find('p')
-            if ne:
-                info['away_team'] = _cl(ne.get_text())
+            if ne: info['away_team'] = _cl(ne.get_text())
             lo = ae.select_one('img')
-            if lo:
-                info['away_logo'] = lo.get('src', '')
-
+            if lo: info['away_logo'] = lo.get('src', '')
         sc = tel.select_one('.score')
         if sc:
             parts = [_cl(p.get_text()) for p in sc.select('p')]
-            if len(parts) >= 2:
-                info['score'] = f"{parts[0]} - {parts[1]}"
+            if len(parts) >= 2: info['score'] = f"{parts[0]} - {parts[1]}"
             lb = sc.select_one('.label')
-            if lb:
-                info['status_label'] = _cl(lb.get_text())
+            if lb: info['status_label'] = _cl(lb.get_text())
 
     if info.get('home_team') and info.get('away_team'):
-        result['info'] = info
-        result['found'] = True
-        result['sections'].append('info')
+        result['info'] = info; result['found'] = True; result['sections'].append('info')
     else:
         return fetch_match_detail(event_id)
 
     mi = sp.select_one('.match-info')
     if mi:
         te = mi.select_one('.times, li')
-        if te:
-            info.setdefault('datetime', _cl(te.get_text()))
+        if te: info.setdefault('datetime', _cl(te.get_text()))
 
     events = parse_events(sp)
-    if events:
-        result['events'] = events
-        result['sections'].append('events')
+    if events: result['events'] = events; result['sections'].append('events')
 
     pred = sp.select_one('.prediction-card')
     if pred:
@@ -454,18 +341,15 @@ def fetch_match_detail_by_url(url: str) -> dict:
                 pred_data['home_name'] = _cl(teams[0].select_one('.team-name').get_text()) if teams[0].select_one('.team-name') else ''
                 pred_data['away_name'] = _cl(teams[1].select_one('.team-name').get_text()) if teams[1].select_one('.team-name') else ''
                 divider = team_info.select_one('.divider')
-                if divider:
-                    pred_data['result'] = _cl(divider.get_text())
+                if divider: pred_data['result'] = _cl(divider.get_text())
         vote_count = pred.select_one('.vote-count')
-        if vote_count:
-            pred_data['vote_count'] = _cl(vote_count.get_text())
+        if vote_count: pred_data['vote_count'] = _cl(vote_count.get_text())
         result['prediction'] = pred_data
 
     try:
         ar = requests.get(
             f"https://bongda.com.vn/api/fixtures/h2h-stats?event_id={event_id}",
-            headers=API_HEADERS,
-            timeout=10
+            headers=API_HEADERS, timeout=10
         )
         if ar.status_code == 200:
             ad = ar.json()
@@ -476,14 +360,8 @@ def fetch_match_detail_by_url(url: str) -> dict:
                     cells = row.select('td, span, p')
                     if len(cells) >= 3:
                         lb = _cl(cells[0].get_text())
-                        if lb:
-                            ast[lb] = {
-                                'home': _cl(cells[1].get_text()),
-                                'away': _cl(cells[2].get_text()),
-                            }
-                if ast:
-                    result['h2h_stats_parsed'] = ast
-                    result['sections'].append('h2h_stats')
+                        if lb: ast[lb] = {'home': _cl(cells[1].get_text()), 'away': _cl(cells[2].get_text())}
+                if ast: result['h2h_stats_parsed'] = ast; result['sections'].append('h2h_stats')
     except Exception:
         pass
 
@@ -508,9 +386,7 @@ def fetch_match_detail_by_url(url: str) -> dict:
                     'ga': _cl(cells[6].get_text()) if len(cells) > 6 else '',
                     'points': _cl(cells[8].get_text()) if len(cells) > 8 else '',
                 })
-        if h2h_data:
-            result['h2h_standings'] = h2h_data
-            result['sections'].append('h2h_standings')
+        if h2h_data: result['h2h_standings'] = h2h_data; result['sections'].append('h2h_standings')
 
     recent_matches = []
     matches_list = sp.select_one('.matches-list')
@@ -529,8 +405,6 @@ def fetch_match_detail_by_url(url: str) -> dict:
                     'away': _cl(away_el.get_text()) if away_el else '',
                     'score': _cl(score_el.get_text()) if score_el else 'vs',
                 })
-        if recent_matches:
-            result['recent_matches'] = recent_matches
-            result['sections'].append('recent')
+        if recent_matches: result['recent_matches'] = recent_matches; result['sections'].append('recent')
 
     return result