Fix VnEgo (use /video page), DanTri always OG image, frontend DanTri in-app read

main.py

@@ -25,8 +25,8 @@ def _get(url,headers=None):
     h=headers or HEADERS;r=requests.get(url,headers=h,timeout=15);r.encoding="utf-8"
     return BeautifulSoup(r.text,"lxml")
 
-# ===== DANTRI =====
 def scrape_dantri_hot():
+    """Scrape DanTri tin nong - always fetch OG image for each article."""
     try:
         soup=_get("https://dantri.com.vn/tin-nong.htm")
         arts=[];seen=set()
@@ -37,36 +37,27 @@ def scrape_dantri_hot():
             if href in seen or not href.endswith(".htm"):continue
             if href=="https://dantri.com.vn/tin-nong.htm":continue
             seen.add(href)
-            img=a.find("img")
-            if not img and a.parent:img=a.parent.find("img")
-            img_src=""
-            if img:img_src=img.get("data-src") or img.get("src","")
-            if img_src and("base64" in img_src or len(img_src)<10):img_src=""
-            arts.append({"title":title,"link":href,"img":img_src,"source":"dantri"})
-            if len(arts)>=15:break
+            arts.append({"title":title,"link":href,"img":"","source":"dantri"})
+            if len(arts)>=12:break
+        # Always fetch OG image for ALL DanTri articles
         def _og(art):
-            if art["img"]:return
             try:
                 r2=requests.get(art["link"],headers=HEADERS,timeout=8);r2.encoding="utf-8"
                 og=BeautifulSoup(r2.text,"lxml").find("meta",property="og:image")
                 art["img"]=og.get("content","") if og else ""
             except:pass
-        need=[a for a in arts if not a["img"]][:5]
-        if need:
-            with ThreadPoolExecutor(3) as ex:list(ex.map(_og,need))
-        return arts[:12]
+        with ThreadPoolExecutor(4) as ex:list(ex.map(_og,arts[:10]))
+        return [a for a in arts if a["img"]][:10]
     except:return[]
 
 def scrape_dantri_article(url):
-    """Parse dantri.com.vn article for in-app reading."""
     try:
         r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8"
         soup=BeautifulSoup(r.text,"lxml")
         for tag in soup.find_all(["script","style","nav","footer","aside"]):tag.decompose()
         h1=soup.find("h1")
         og=soup.find("meta",property="og:image");og_img=og.get("content","") if og else ""
-        # DanTri content selectors
-        content=soup.select_one("div.singular-content") or soup.select_one("div.e-magazine__body") or soup.select_one("article.e-magazine") or soup.select_one("div.detail-content")
+        content=soup.select_one("div.singular-content") or soup.select_one("div.e-magazine__body") or soup.select_one("article")
         body=[]
         if content:
             for el in content.find_all(["p","h2","h3","figure","img"],recursive=True):
@@ -81,7 +72,6 @@ def scrape_dantri_article(url):
                     if im:
                         s=im.get("data-src") or im.get("src","")
                         if s and"base64" not in s and len(s)>10:body.append({"type":"img","src":s,"alt":im.get("alt","")})
-        # Fallback: find all p in page if content block empty
         if not body:
             for p in soup.find_all("p"):
                 t=p.get_text(strip=True)
@@ -93,32 +83,50 @@ def scrape_dantri_article(url):
         return{"title":h1.get_text(strip=True) if h1 else "","summary":desc,"og_image":og_img,"body":body,"source":"dantri","url":url}
     except:return None
 
-# ===== VNEXPRESS VIDEO (VnEgo) =====
 def scrape_vne_video():
-    """Scrape VnExpress VnEgo section."""
+    """Scrape VnExpress video page (server-rendered, not SPA like /vne-go)."""
     try:
-        soup=_get("https://vnexpress.net/vne-go")
+        soup=_get("https://vnexpress.net/video")
         arts=[];seen=set()
-        for a in soup.find_all("a",href=True):
+        for it in soup.select("article.item-news")[:15]:
+            a=it.select_one("a[href]")
+            if not a:continue
             href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
-            if not title or len(title)<8 or"javascript:" in href:continue
-            if not href.startswith("http"):continue
-            if href in seen:continue
-            if"/video/" not in href and"/vne-go/" not in href:continue
+            if not title or len(title)<8 or href in seen:continue
             seen.add(href)
-            img=a.find("img")
-            if not img:
-                parent=a.parent
-                if parent:img=parent.find("img")
-            img_src=""
-            if img:
-                img_src=img.get("data-src") or img.get("src","")
+            im=it.find("img");img_src=""
+            if im:
+                img_src=im.get("data-src") or im.get("src","")
                 if img_src and"blank" in img_src:
-                    source=img.find_next("source") or (a.find("source") if a else None)
+                    source=it.find("source")
                     if source:img_src=source.get("srcset","").split(",")[0].strip().split(" ")[0]
-            if not img_src or len(img_src)<10:img_src=""
             arts.append({"title":title,"link":href,"img":img_src,"source":"vne-video"})
-            if len(arts)>=15:break
+        # Fallback: if article.item-news not found, try generic links
+        if not arts:
+            for a in soup.find_all("a",href=True):
+                href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
+                if not title or len(title)<10:continue
+                if not href.startswith("http") or href in seen:continue
+                if"/video/" not in href:continue
+                seen.add(href)
+                img=a.find("img")
+                if not img and a.parent:img=a.parent.find("img")
+                img_src=""
+                if img:img_src=img.get("data-src") or img.get("src","")
+                if img_src and("blank" in img_src or"base64" in img_src):img_src=""
+                arts.append({"title":title,"link":href,"img":img_src,"source":"vne-video"})
+                if len(arts)>=12:break
+        # Fetch OG for missing images
+        def _og(art):
+            if art["img"]:return
+            try:
+                r2=requests.get(art["link"],headers=HEADERS,timeout=8);r2.encoding="utf-8"
+                og=BeautifulSoup(r2.text,"lxml").find("meta",property="og:image")
+                art["img"]=og.get("content","") if og else ""
+            except:pass
+        need=[a for a in arts if not a["img"]][:5]
+        if need:
+            with ThreadPoolExecutor(3) as ex:list(ex.map(_og,need))
         return arts[:12]
     except:return[]
 
@@ -280,7 +288,6 @@ def extract_bdp_video(url):
         return{"src":source.get("src","") if source else "","poster":video.get("poster","")}
     except:return None
 
-# All VnExpress categories
 VNE_CATS={"thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),"cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe"),"giao-duc":("https://vnexpress.net/giao-duc","Giáo Dục"),"phap-luat":("https://vnexpress.net/phap-luat","Pháp Luật"),"du-lich":("https://vnexpress.net/du-lich","Du Lịch"),"xe":("https://vnexpress.net/oto-xe-may","Xe"),"doi-song":("https://vnexpress.net/doi-song","Đời Sống")}
 
 @app.get("/api/homepage")
@@ -288,7 +295,6 @@ def api_homepage():
     def _f():
         articles=[]
         with ThreadPoolExecutor(8) as ex:
-            # Load ALL VNE categories for homepage
             futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in VNE_CATS}
             futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
             for f in as_completed(futs):