Fix article reader for vatvostudio/bongda/bongdaplus: clean body (strip nav/ads/related), extract cover + inline images

app_v2_entry.py

@@ -167,31 +167,74 @@ def _search_all(topic,limit=36):
     return out[:limit]
 # Override article endpoint
 app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/api/article' and 'GET' in getattr(r,'methods',set()))]
+_JINA_BOILER_STOP=['facebooktelegram','copy link','follow us','bài viết liên quan','tin liên quan',
+    'cùng chuyên mục','bình luận','trả lời','bản quyền','địa chỉ:','lưu ý:','đăng ký nhận tin',
+    'chia sẻ bài viết','tags:','© ','bongdaplus.vn.','vật vờ studio để cập nhật','mời các bạn theo dõi',
+    'cập nhật tin tức, thủ thuật']
+_JINA_NAV=['đăng nhập','tạo tài khoản','reading:','sponsored by']
+_JINA_IMG_DENY=['logo','avatar','sponsor','vvaba','ductrinh','longnguyen','hoangtrinh','vvs_logo',
+    'cropped-','login.png','/icon','placeholder','/ads/','banner','giaminhmedia','-330x220','-420x280',
+    '-150x','-100x','thumb']
+_JINA_AD_DENY=['giaminhmedia','delivery/cl.php','[hot]','[video','săn ngay','giảm giá','khuyến mãi',
+    'mua ngay','đặt mua','shopee','lazada','tiki.vn']
+def _jina_strip(s):
+    s=re.sub(r'!\[[^\]]*\]\([^)]*\)','',s)
+    s=re.sub(r'\[([^\]]*)\]\([^)]*\)',r'\1',s)
+    s=re.sub(r'</?[a-zA-Z][^>]*>','',s)
+    s=re.sub(r'[#>*`_]+','',s)
+    return re.sub(r'\s+',' ',s).strip()
+def _jina_good_img(u,alt=''):
+    ul=u.lower();al=(alt or '').lower()
+    if any(k in ul for k in _JINA_IMG_DENY):return False
+    if 'vật vờ studio' in al or 'sponsored' in al:return False
+    if not re.search(r'\.(jpg|jpeg|png|webp|gif)',ul) and 'imgthumbnail' not in ul and '/media/' not in ul.lower():return False
+    return True
 def _jina_article(url):
-    """Read an article via r.jina.ai (for Cloudflare/SPA sites: vatvostudio, bongdaplus)."""
+    """Read ONLY the main article body via r.jina.ai (Cloudflare/SPA sites: vatvostudio, bongdaplus, bongda).
+    Strips navigation menus, related-article lists, comments, ads and footer; keeps cover + inline images."""
     try:
         r=req.get("https://r.jina.ai/"+url,headers={'Accept':'text/markdown','X-Return-Format':'markdown','User-Agent':'Mozilla/5.0'},timeout=35)
         if r.status_code!=200 or not r.text:return None
-        lines=[x.rstrip() for x in r.text.splitlines()]
-        title='';og_img='';summary='';body=[]
-        for ln in lines[:40]:
-            if ln.startswith('Title:') and not title:title=ln.replace('Title:','',1).strip()
-        # first content image
-        for ln in lines:
-            mi=re.search(r'!\[[^\]]*\]\((https?://[^)]+)\)',ln)
-            if mi and 'wp-content' in mi.group(1) or (mi and 'media' in mi.group(1)):og_img=mi.group(1);break
-        intxt=False
-        for ln in lines:
-            if ln.startswith('Markdown Content:'):intxt=True;continue
-            if not intxt:continue
-            t=re.sub(r'!\[[^\]]*\]\([^)]+\)','',ln)            # drop images
-            t=re.sub(r'\[([^\]]+)\]\([^)]+\)',r'\1',t)         # unwrap links
-            t=re.sub(r'[#>*_`]+','',t).strip()
-            if len(t)>=40:body.append({'type':'p','text':t})
-            if len(body)>=60:break
-        if not title and body:title=body[0]['text'][:90]
-        if not body:return None
-        return {'title':_clean(title),'summary':_clean(body[0]['text'][:200]) if body else '','og_image':og_img,'body':body,'source':'jina','url':url}
+        lines=r.text.splitlines()
+        title=''
+        for ln in lines[:30]:
+            if ln.startswith('Title:'):title=ln.replace('Title:','',1).strip();break
+        tw=set(w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title.lower()) if len(w)>3)
+        # body starts after the LAST markdown H1 matching the title (= the real headline)
+        start=0
+        for i,ln in enumerate(lines):
+            if ln.startswith('# '):
+                hw=set(re.findall(r'[A-Za-zÀ-ỹ0-9]+',_jina_strip(ln).lower()))
+                if tw and len(tw&hw)>=max(2,len(tw)//2):start=i+1
+        body=[];og_img=''
+        for ln in lines[start:]:
+            low=_jina_strip(ln).lower()
+            if low and any(b in low for b in _JINA_BOILER_STOP):break  # related/footer -> stop
+            if any(a in ln.lower() for a in _JINA_AD_DENY):continue     # skip ads/video embeds
+            mi=re.search(r'!\[([^\]]*)\]\((https?://[^)\s]+)\)',ln)
+            if mi and _jina_good_img(mi.group(2),mi.group(1)):
+                src=mi.group(2)
+                if not og_img:og_img=src
+                body.append({'type':'img','src':src});continue
+            s=ln.strip();is_h=bool(re.match(r'^#{2,4}\s',s));txt=_jina_strip(ln)
+            if not txt or len(txt)<40:continue
+            if any(p in txt.lower() for p in _JINA_NAV):continue
+            raw=re.sub(r'!\[[^\]]*\]\([^)]*\)','',ln);lt=''.join(re.findall(r'\[([^\]]*)\]\([^)]*\)',raw))
+            if len(lt)>len(txt)*0.5:continue  # mostly link -> nav junk
+            body.append({'type':'heading' if is_h else 'p','text':txt})
+            if len([b for b in body if b['type']!='img'])>=40:break
+        if not og_img:
+            for ln in lines:
+                mi=re.search(r'!\[([^\]]*)\]\((https?://[^)\s]+)\)',ln)
+                if mi and _jina_good_img(mi.group(2),mi.group(1)):og_img=mi.group(2);break
+        out=[]
+        for b in body:
+            if out and b==out[-1]:continue
+            out.append(b)
+        if not title and out:title=next((b['text'] for b in out if b['type']=='p'),'')[:90]
+        if not out:return None
+        first_p=next((b['text'] for b in out if b['type']=='p'),'')
+        return {'title':_clean(title),'summary':_clean(first_p[:200]),'og_image':og_img,'image':og_img,'body':out,'source':'jina','url':url}
     except:return None
 def _scrape_generic(url):
     try: