v15.38: Fix scrape_vne_article — fallback to _scrape_generic, try Windows UA

main.py

@@ -616,46 +616,53 @@ def _scrape_generic(url):
 
 def scrape_vne_article(url):
     """Scrape VnExpress article."""
+    data=_scrape_generic(url)
+    if data and data.get('body') and len(data['body'])>2:
+        data['source']='vnexpress'
+        return data
+    # If generic didn't work well, try specific VnExpress selectors
     try:
-        r=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15)
+        r=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15)
         r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','.banner-ads','iframe','.fb-comments','.fb-root']):tag.decompose()
-        h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
-        title=(h1.get_text(strip=True) if h1 else '') or (ogt.get('content','') if ogt else '')
-        ogd=soup.find('meta',property='og:description');summary=ogd.get('content','') if ogd else ''
-        ogi=soup.find('meta',property='og:image');og_img=ogi.get('content','') if ogi else ''
-        article=soup.select_one('.fck_detail,.sidebar-1,article')
-        if not article:return _scrape_generic(url)
-        for tag in article.find_all(['script','style','.ads','iframe']):tag.decompose()
-        body=[]
-        seen_imgs=set()
-        for el in article.find_all(['p','h2','h3','figure','img'],recursive=True):
-            if el.name=='p':
-                t=el.get_text(strip=True)
-                if t and len(t)>10:body.append({'type':'p','text':t})
-            elif el.name in('h2','h3'):
-                t=el.get_text(strip=True)
-                if t:body.append({'type':'heading','text':t})
-            elif el.name=='figure':
-                im=el.find('img')
-                if im:
-                    src=im.get('data-src') or im.get('src') or''
+        article=soup.select_one('.fck_detail,.sidebar-1')
+        if article and len(article.find_all('p'))>=3:
+            for tag in article.find_all(['script','style','.ads','iframe']):tag.decompose()
+            body=[]
+            seen_imgs=set()
+            for el in article.find_all(['p','h2','h3','figure','img'],recursive=True):
+                if el.name=='p':
+                    t=el.get_text(strip=True)
+                    if t and len(t)>10:body.append({'type':'p','text':t})
+                elif el.name in('h2','h3'):
+                    t=el.get_text(strip=True)
+                    if t:body.append({'type':'heading','text':t})
+                elif el.name=='figure':
+                    im=el.find('img')
+                    if im:
+                        src=im.get('data-src') or im.get('src') or''
+                        if src and src not in seen_imgs and'base64' not in src:
+                            seen_imgs.add(src)
+                            if src.startswith('//'):src='https:'+src
+                            body.append({'type':'img','src':src})
+                    cap=el.find('figcaption')
+                    if cap:
+                        t=cap.get_text(strip=True)
+                        if t:body.append({'type':'p','text':t})
+                elif el.name=='img':
+                    src=el.get('data-src') or el.get('src') or''
                     if src and src not in seen_imgs and'base64' not in src:
                         seen_imgs.add(src)
                         if src.startswith('//'):src='https:'+src
                         body.append({'type':'img','src':src})
-                cap=el.find('figcaption')
-                if cap:
-                    t=cap.get_text(strip=True)
-                    if t:body.append({'type':'p','text':t})
-            elif el.name=='img':
-                src=el.get('data-src') or el.get('src') or''
-                if src and src not in seen_imgs and'base64' not in src:
-                    seen_imgs.add(src)
-                    if src.startswith('//'):src='https:'+src
-                    body.append({'type':'img','src':src})
-        return{'title':title,'summary':summary,'og_image':og_img,'body':body[:50],'source':'vnexpress','url':url}
-    except:return _scrape_generic(url)
+            if body:
+                h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
+                title=(h1.get_text(strip=True) if h1 else '') or (ogt.get('content','') if ogt else '')
+                ogd=soup.find('meta',property='og:description');summary=ogd.get('content','') if ogd else ''
+                ogi=soup.find('meta',property='og:image');og_img=ogi.get('content','') if ogi else ''
+                return{'title':title,'summary':summary,'og_image':og_img,'body':body[:50],'source':'vnexpress','url':url}
+    except:pass
+    return data
 
 def scrape_dantri_article(url):
     """Scrape Dân Trí article."""