Fix fixtures download headers + add github fallback + fix news image extraction"

wc2026_scraper.py

@@ -1,9 +1,8 @@
 """
-World Cup 2026 Data - CORRECT official data
-- Fixtures: fixturedownload.com JSON (104 matches, stadiums, times) + convert UTC→VN
-- BXH: bongda.com.vn API tournament_id=24254
-- News: with images from VnExpress + Thanh Niên + TT&VH
-- Livescore: when match is live, score updates from bongda API
+World Cup 2026 Data
+- Fixtures: fixturedownload.com JSON (with Referer header) + GitHub fallback
+- BXH: bongda.com.vn API
+- News: VnExpress + Thanh Niên + Dân Trí (with proper image extraction)
 """
 import requests, re, time, threading
 from bs4 import BeautifulSoup
@@ -18,7 +17,7 @@ BONGDA_HEADERS = {
     "Referer": "https://bongda.com.vn/giai-dau/24254/standings/world-cup",
     "X-Requested-With": "XMLHttpRequest"
 }
-UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'}
 WC_ID = 24254
 VN_TZ = timezone(timedelta(hours=7))
 CACHE = {}
@@ -49,47 +48,59 @@ def _fetch(url, timeout=15):
     except: return ''
 
 def _utc_to_vn(date_str):
-    """Convert UTC datetime string to Vietnam time string."""
     try:
-        if 'T' in date_str:
-            dt = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
+        if not date_str: return ''
+        if 'T' in str(date_str):
+            dt = datetime.fromisoformat(str(date_str).replace('Z', '+00:00'))
             if dt.tzinfo is None:
                 dt = dt.replace(tzinfo=timezone.utc)
             vn = dt.astimezone(VN_TZ)
             return vn.strftime('%H:%M %d/%m/%Y')
-        return date_str
+        return str(date_str)
     except:
-        return date_str
+        return str(date_str)
 
-# ==================== FIXTURES (JSON chính xác 104 trận) ====================
+# ==================== FIXTURES ====================
 def scrape_fixtures():
-    """Get WC2026 fixtures from fixturedownload.com JSON - OFFICIAL 104 matches."""
-    c = _cached('wc_fix', 3600)  # 1h cache - data doesn't change often
+    c = _cached('wc_fix', 3600)
     if c is not None: return c
 
     matches = []
-    try:
-        r = requests.get('https://fixturedownload.com/download/json/fifa-world-cup-2026', headers=UA, timeout=15)
-        if r.status_code == 200:
-            data = r.json()
-            for m in data:
-                matches.append({
-                    'match_number': m.get('MatchNumber', ''),
-                    'round': m.get('RoundNumber', ''),
-                    'date_vn': _utc_to_vn(m.get('DateUtc', '')),
-                    'date_utc': m.get('DateUtc', ''),
-                    'location': m.get('Location', ''),
-                    'home': m.get('HomeTeam', ''),
-                    'away': m.get('AwayTeam', ''),
-                    'group': m.get('Group', ''),
-                    'home_score': m.get('HomeTeamScore'),
-                    'away_score': m.get('AwayTeamScore'),
-                })
-    except: pass
+    urls = [
+        ('https://fixturedownload.com/download/json/fifa-world-cup-2026', {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+            'Accept': 'application/json, text/plain, */*',
+            'Referer': 'https://fixturedownload.com/view/json/fifa-world-cup-2026'
+        }),
+        ('https://raw.githubusercontent.com/mjwebmaster/world-cup-2026-schedule-data/main/world-cup-2026.json', UA),
+    ]
 
-    r = {'matches': matches, 'total': len(matches)}
-    _set('wc_fix', r)
-    return r
+    for url, headers in urls:
+        if matches: break
+        try:
+            r = requests.get(url, headers=headers, timeout=20)
+            if r.status_code == 200:
+                data = r.json()
+                if isinstance(data, list) and len(data) > 10:
+                    for m in data:
+                        date_raw = m.get('DateUtc') or m.get('date_utc') or m.get('date') or ''
+                        matches.append({
+                            'match_number': m.get('MatchNumber') or m.get('match_number') or '',
+                            'round': m.get('RoundNumber') or m.get('round') or '',
+                            'date_vn': _utc_to_vn(date_raw),
+                            'date_utc': date_raw,
+                            'location': m.get('Location') or m.get('location') or m.get('venue') or '',
+                            'home': m.get('HomeTeam') or m.get('home_team') or m.get('home') or '',
+                            'away': m.get('AwayTeam') or m.get('away_team') or m.get('away') or '',
+                            'group': m.get('Group') or m.get('group') or m.get('stage') or '',
+                            'home_score': m.get('HomeTeamScore') or m.get('home_score'),
+                            'away_score': m.get('AwayTeamScore') or m.get('away_score'),
+                        })
+        except: continue
+
+    result = {'matches': matches, 'total': len(matches)}
+    _set('wc_fix', result)
+    return result
 
 # ==================== BXH ====================
 def scrape_standings():
@@ -118,13 +129,34 @@ def scrape_lineups(event_id): return {'html': _bongda(f"/api/fixtures/lineups?ev
 def scrape_match_detail(event_id): return {'html': _bongda(f"/api/fixtures/commentaries?event_id={event_id}")}
 def scrape_summary(): return scrape_standings()
 
-# ==================== NEWS (with images!) ====================
+# ==================== NEWS ====================
+def _get_img_from_article(art):
+    """Extract image URL from a VnExpress/generic article element."""
+    # Try img tag
+    for img in art.select('img'):
+        src = img.get('data-src') or img.get('data-original') or img.get('src') or ''
+        if src and 'blank' not in src and 'data:image' not in src and len(src) > 20:
+            if src.startswith('//'): src = 'https:' + src
+            return src
+    # Try source srcset (VnExpress picture element)
+    for source in art.select('source[srcset]'):
+        srcset = source.get('srcset', '')
+        if srcset:
+            first = srcset.split(',')[0].strip().split(' ')[0]
+            if first and len(first) > 20:
+                if first.startswith('//'): first = 'https:' + first
+                return first
+    # Try background-image in style
+    for el in art.select('[style*="background"]'):
+        m = re.search(r'url\(["\']?([^"\')\s]+)', el.get('style', ''))
+        if m: return m.group(1)
+    return ''
+
 def scrape_wc_news():
-    """Get WC2026 news WITH images from multiple sources."""
     c = _cached('wc_news', 300)
     if c is not None: return c
     news = []
-    # VnExpress (always has og:image in article items)
+    # VnExpress
     try:
         html = _fetch(f'https://timkiem.vnexpress.net/?q={quote("World Cup 2026")}')
         if html:
@@ -134,42 +166,28 @@ def scrape_wc_news():
                 if not a: continue
                 title = _clean(a.get('title', '') or a.get_text())
                 href = a.get('href', '')
-                # Get image - VnExpress uses data-src or source srcset
-                img = ''
-                img_el = art.select_one('img')
-                if img_el:
-                    img = img_el.get('data-src', '') or img_el.get('src', '')
-                if not img:
-                    source_el = art.select_one('source')
-                    if source_el:
-                        srcset = source_el.get('srcset', '')
-                        if srcset: img = srcset.split(',')[0].strip().split(' ')[0]
-                if not img:
-                    # Try picture element
-                    pic = art.select_one('picture img, .thumb-art img')
-                    if pic: img = pic.get('data-src', '') or pic.get('src', '')
+                img = _get_img_from_article(art)
                 if title and href:
                     news.append({'title': title, 'link': href, 'img': img, 'source': 'VnExpress'})
     except: pass
-    # Thanh Niên WC2026
+    # Thanh Niên
     try:
         html = _fetch('https://worldcup2026.thanhnien.vn/')
         if html:
             soup = BeautifulSoup(html, 'lxml')
-            for item in soup.select('.story, .news-item, article, .item')[:12]:
+            for item in soup.select('.story, .news-item, article, .item, .zone--timeline .story')[:12]:
                 a = item.select_one('a[href]')
                 if not a: continue
                 title = _clean(a.get('title', '') or a.get_text())
                 if not title or len(title) < 15: continue
                 href = a.get('href', '')
                 if not href.startswith('http'): href = 'https://worldcup2026.thanhnien.vn' + href
-                img_el = item.select_one('img')
-                img = (img_el.get('data-src', '') or img_el.get('src', '')) if img_el else ''
+                img = _get_img_from_article(item)
                 if href not in [n['link'] for n in news]:
                     news.append({'title': title, 'link': href, 'img': img, 'source': 'Thanh Niên'})
                 if len(news) >= 20: break
     except: pass
-    # Dân Trí WC
+    # Dân Trí
     try:
         html = _fetch('https://dantri.com.vn/the-thao/world-cup.htm')
         if html:
@@ -180,8 +198,10 @@ def scrape_wc_news():
                 title = _clean(a.get('title', '') or a.get_text())
                 href = a.get('href', '')
                 if not href.startswith('http'): href = 'https://dantri.com.vn' + href
-                img_el = art.select_one('img')
-                img = (img_el.get('data-src', '') or img_el.get('src', '')) if img_el else ''
+                img = _get_img_from_article(art)
+                # Proxy dantri images
+                if img and 'cdnphoto.dantri' in img:
+                    img = '/api/proxy/img?url=' + quote(img, safe='')
                 if title and len(title) > 15 and href not in [n['link'] for n in news]:
                     news.append({'title': title, 'link': href, 'img': img, 'source': 'Dân Trí'})
     except: pass
@@ -202,11 +222,7 @@ def scrape_road_to_wc():
                 if not a: continue
                 title = _clean(a.get('title', '') or a.get_text())
                 href = a.get('href', '')
-                img_el = art.select_one('img')
-                img = (img_el.get('data-src', '') or img_el.get('src', '')) if img_el else ''
-                if not img:
-                    source_el = art.select_one('source')
-                    if source_el: img = source_el.get('srcset', '').split(',')[0].strip().split(' ')[0]
+                img = _get_img_from_article(art)
                 if title and href and href not in [x['link'] for x in articles]:
                     articles.append({'title': title, 'link': href, 'img': img, 'source': 'VnExpress', 'type': 'road'})
         except: continue