Fix: road_to_wc always gets og:image; hot_topics from 8 RSS sources (not just VnExpress)"

wc2026_scraper.py

@@ -1,5 +1,5 @@
 """
-World Cup 2026 Data Module - NEWS ALWAYS HAS IMAGES (fetches og:image server-side)
+World Cup 2026 Data Module - ALL NEWS ITEMS ALWAYS HAVE IMAGES
 """
 import requests, re, time, threading
 from bs4 import BeautifulSoup
@@ -45,15 +45,13 @@ def _fetch(url, timeout=12):
     except: return ''
 
 def _get_og_image(url):
-    """Fetch og:image from a URL — guaranteed to get thumbnail."""
+    """Fetch og:image from URL."""
     try:
-        r = requests.get(url, headers=UA, timeout=8, allow_redirects=True)
+        r = requests.get(url, headers=UA, timeout=10, allow_redirects=True)
         r.encoding = 'utf-8'
         if r.status_code != 200: return ''
-        # Quick regex for og:image (faster than full parse)
-        m = re.search(r'<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\']+)["\']', r.text)
-        if not m:
-            m = re.search(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image["\']', r.text)
+        m = re.search(r'<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\']+)["\']', r.text[:5000])
+        if not m: m = re.search(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image["\']', r.text[:5000])
         if m:
             img = m.group(1)
             if img.startswith('//'): img = 'https:' + img
@@ -114,7 +112,6 @@ def scrape_standings():
     if not html: html = _bongda(f"/api/league-table/home?tournament_id={WC_ID}")
     r = {'html': html}; _set('wc_bxh', r); return r
 
-# ==================== STATS ====================
 def scrape_stats():
     c = _cached('wc_stats', 600)
     if c is not None: return c
@@ -127,13 +124,12 @@ def scrape_lineups(event_id): return {'html': _bongda(f"/api/fixtures/lineups?ev
 def scrape_match_detail(event_id): return {'html': _bongda(f"/api/fixtures/commentaries?event_id={event_id}")}
 def scrape_summary(): return scrape_standings()
 
-# ==================== NEWS (ALWAYS fetch og:image for each article) ====================
+# ==================== NEWS ====================
 def scrape_wc_news():
     c = _cached('wc_news', 300)
     if c is not None: return c
     news = []
-    
-    # VnExpress World Cup (best images)
+    # VnExpress
     try:
         html = _fetch(f'https://timkiem.vnexpress.net/?q={quote("World Cup 2026")}')
         if html:
@@ -146,11 +142,9 @@ def scrape_wc_news():
                 if not title or href in [n['link'] for n in news]: continue
                 img_el = art.select_one('img')
                 img = (img_el.get('data-src', '') or img_el.get('src', '')) if img_el else ''
-                # Filter blank/placeholder images
                 if img and ('blank' in img or 'data:image' in img): img = ''
                 news.append({'title': title, 'link': href, 'img': img, 'source': 'VnExpress'})
     except: pass
-    
     # Thanh Niên
     try:
         html = _fetch('https://worldcup2026.thanhnien.vn/')
@@ -167,7 +161,6 @@ def scrape_wc_news():
                 news.append({'title': title, 'link': href, 'img': img, 'source': 'Thanh Niên'})
                 if len(news) >= 20: break
     except: pass
-    
     # Dân Trí
     try:
         html = _fetch('https://dantri.com.vn/the-thao/world-cup.htm')
@@ -180,7 +173,6 @@ def scrape_wc_news():
                 if not title or len(title)<15 or href in [n['link'] for n in news]: continue
                 news.append({'title': title, 'link': href, 'img': '', 'source': 'Dân Trí'})
     except: pass
-    
     # TT&VH
     try:
         html = _fetch('https://thethaovanhoa.vn/rss/world-cup-2026.rss')
@@ -199,28 +191,25 @@ def scrape_wc_news():
                 news.append({'title': title, 'link': link, 'img': img, 'source': 'TT&VH'})
     except: pass
     
-    # === FETCH og:image for items without images (parallel, fast) ===
-    def _fill_img(item):
-        if item.get('img'): return item
-        img = _get_og_image(item['link'])
-        if img: item['img'] = img
-        return item
-    
-    # Only fetch for first 15 items to keep response fast
-    items_need_img = [n for n in news[:15] if not n.get('img')]
-    if items_need_img:
-        with ThreadPoolExecutor(5) as ex:
-            futs = [ex.submit(_fill_img, item) for item in items_need_img[:10]]
-            for f in as_completed(futs, timeout=12):
-                try: f.result()
-                except: pass
+    # FETCH og:image for ALL items without images
+    def _fill(item):
+        if item.get('img'): return
+        item['img'] = _get_og_image(item['link'])
+    with ThreadPoolExecutor(6) as ex:
+        futs = [ex.submit(_fill, n) for n in news[:20] if not n.get('img')]
+        for f in as_completed(futs, timeout=15):
+            try: f.result()
+            except: pass
     
     _set('wc_news', news[:30]); return news[:30]
 
+# ==================== ROAD TO WC (also fetches og:image for ALL) ====================
 def scrape_road_to_wc():
     c = _cached('wc_road', 600)
     if c is not None: return c
     articles = []
+    
+    # VnExpress
     for q in ['đường tới World Cup 2026', 'tuyển Việt Nam World Cup 2026']:
         try:
             html = _fetch(f'https://timkiem.vnexpress.net/?q={quote(q)}')
@@ -233,10 +222,37 @@ def scrape_road_to_wc():
                 href = a.get('href', '')
                 img_el = art.select_one('img')
                 img = (img_el.get('data-src', '') or img_el.get('src', '')) if img_el else ''
-                if not img: img = _get_og_image(href)
+                if img and ('blank' in img or 'data:image' in img): img = ''
                 if title and href and href not in [x['link'] for x in articles]:
                     articles.append({'title': title, 'link': href, 'img': img, 'source': 'VnExpress', 'type': 'road'})
         except: continue
+    
+    # Thanh Niên road
+    try:
+        html = _fetch('https://worldcup2026.thanhnien.vn/')
+        if html:
+            soup = BeautifulSoup(html, 'lxml')
+            for a in soup.select('a[href*="duong-toi"], a[href*="vong-loai"], a[href*="tuyen-viet-nam"]')[:5]:
+                href = a.get('href', '')
+                title = _clean(a.get('title', '') or a.get_text())
+                if not title or len(title) < 15: continue
+                if not href.startswith('http'): href = 'https://worldcup2026.thanhnien.vn' + href
+                if href in [x['link'] for x in articles]: continue
+                img_el = a.select_one('img') or (a.parent.select_one('img') if a.parent else None)
+                img = (img_el.get('data-src','') or img_el.get('src','')) if img_el else ''
+                articles.append({'title': title, 'link': href, 'img': img, 'source': 'Thanh Niên', 'type': 'road'})
+    except: pass
+    
+    # FETCH og:image for ALL without images
+    def _fill(item):
+        if item.get('img'): return
+        item['img'] = _get_og_image(item['link'])
+    with ThreadPoolExecutor(5) as ex:
+        futs = [ex.submit(_fill, a) for a in articles if not a.get('img')]
+        for f in as_completed(futs, timeout=15):
+            try: f.result()
+            except: pass
+    
     _set('wc_road', articles[:20]); return articles[:20]
 
 # ==================== ALL ====================
@@ -252,7 +268,7 @@ def get_wc2026_all():
             ex.submit(scrape_wc_news): 'news',
             ex.submit(scrape_road_to_wc): 'road',
         }
-        for f in as_completed(futs, timeout=30):
+        for f in as_completed(futs, timeout=35):
             key = futs[f]
             try: data[key] = f.result()
             except: data[key] = {} if key in ('fixtures','standings','stats') else []