Spaces:
Sleeping
Sleeping
| """Final2: improve article-image detection without over-filtering real article images.""" | |
| import re, requests | |
| from urllib.parse import urlparse | |
| import ai_runtime_final as f1 | |
| from ai_runtime_final import app, base, rt, HTMLResponse, JSONResponse, Request, Query | |
| def _domain(url): | |
| try:return urlparse(url or '').netloc.replace('www.','') | |
| except Exception:return '' | |
| def _abs_url(src, base_url): | |
| if not src:return '' | |
| src=src.strip() | |
| if src.startswith('//'):return 'https:'+src | |
| if src.startswith('/'): | |
| try: | |
| p=urlparse(base_url);return f'{p.scheme}://{p.netloc}{src}' | |
| except Exception:return src | |
| return src | |
| BAD_RE=re.compile(r'(related|relate|recommend|suggest|sidebar|ads|advert|popular|xem-them|xemthem|tin-lien-quan|tinlienquan|doc-them|docthem|other-news|news-other|article-related|box-tin|box_related|story-related|recommend-news|same-category|cate-list|most-view|banner|qc|quang-cao|sponsor|social|share|comment|author|newsletter)',re.I) | |
| GOOD_RE=re.compile(r'(article|content|detail|body|post|entry|story|fck|cms|singular|main|news)',re.I) | |
| IMG_EXT_RE=re.compile(r'\.(jpg|jpeg|png|webp|avif)(\?|$)',re.I) | |
| ARTICLE_LINK_RE=re.compile(r'\.(html|htm|shtml|tpo|chn)(\?|$)|/\d{4}/|post\d+|article',re.I) | |
| def _clean_soup(soup): | |
| for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe']): | |
| tag.decompose() | |
| def _find_article_block(soup): | |
| """Find the article body first; do not delete suspected related blocks before finding it.""" | |
| selectors=[ | |
| 'article', 'main article', | |
| '.article-content','.article__content','.article__body','.article-body','.article-detail','.article__detail', | |
| '.detail-content','.content-detail','.singular-content','.news-content','.post-content','.entry-content', | |
| '.knc-content','.fck_detail','.cms-body','.story-body','.maincontent','.main-content', | |
| '[class*=article-content]','[class*=article__content]','[class*=detail-content]','[class*=singular-content]', | |
| '[class*=cms-body]','[class*=story-body]' | |
| ] | |
| for sel in selectors: | |
| el=soup.select_one(sel) | |
| if el and (len(el.find_all('p'))>=2 or len(el.find_all(['figure','picture','img']))>=1): | |
| return el | |
| best=None;best_score=0 | |
| for el in soup.find_all(['article','main','section','div']): | |
| cls=' '.join(el.get('class',[]));eid=el.get('id','') | |
| if BAD_RE.search(cls+' '+eid) and not GOOD_RE.search(cls+' '+eid): | |
| continue | |
| ps=el.find_all('p');imgs=el.find_all('img') | |
| text=' '.join(p.get_text(' ',strip=True) for p in ps) | |
| long_ps=sum(1 for p in ps if len(p.get_text(' ',strip=True))>40) | |
| score=long_ps*180+len(ps)*40+min(len(text),5000)+len(imgs)*25 | |
| if GOOD_RE.search(cls+' '+eid):score+=800 | |
| if score>best_score: | |
| best=el;best_score=score | |
| return best or soup | |
| def _ancestor_bad(im, block): | |
| node=im | |
| while node and node is not block: | |
| if getattr(node,'name',None) in ['aside','nav','footer']: | |
| return True | |
| cls=' '.join(node.get('class',[])) if hasattr(node,'get') else '' | |
| eid=node.get('id','') if hasattr(node,'get') else '' | |
| if BAD_RE.search(cls+' '+eid): | |
| return True | |
| node=getattr(node,'parent',None) | |
| return False | |
| def _image_anchor_penalty(im, page_url): | |
| a=im.find_parent('a') | |
| if not a:return 0 | |
| href=_abs_url(a.get('href',''),page_url) | |
| if not href:return 0 | |
| # If anchor opens the image itself, do not penalize. | |
| if IMG_EXT_RE.search(href):return 0 | |
| # If anchor points to another article, it is probably related content. | |
| try: | |
| p1=urlparse(page_url);p2=urlparse(href) | |
| if href!=page_url and ARTICLE_LINK_RE.search(href) and (p2.path!=p1.path): | |
| return -100 | |
| except Exception:pass | |
| return -10 | |
| def _near_article_text_score(im): | |
| score=0 | |
| # caption/figcaption is strong sign of article image | |
| fig=im.find_parent('figure') | |
| if fig: | |
| score+=5 | |
| cap=fig.find('figcaption') | |
| if cap and len(cap.get_text(' ',strip=True))>10:score+=4 | |
| if im.find_parent('picture'):score+=2 | |
| # paragraph around image | |
| parent=im.parent | |
| for node in [parent, getattr(parent,'parent',None) if parent else None, fig]: | |
| if not node:continue | |
| ps=node.find_all('p') if hasattr(node,'find_all') else [] | |
| if any(len(p.get_text(' ',strip=True))>40 for p in ps):score+=3;break | |
| # sibling paragraph near figure/image | |
| holder=fig or parent | |
| if holder: | |
| for sib in [holder.find_previous_sibling(), holder.find_next_sibling()]: | |
| if sib and len(sib.get_text(' ',strip=True))>40: | |
| score+=2 | |
| break | |
| return score | |
| def _image_score(im, src, block, page_url): | |
| low=(src or '').lower() | |
| if not src or src.startswith('data:') or 'base64' in low:return -999 | |
| if any(x in low for x in ['logo','icon','avatar','sprite','tracking','pixel','social','share','author']):return -999 | |
| if _ancestor_bad(im,block):return -999 | |
| score=0 | |
| # Explicit dimensions: only reject truly tiny images; if missing dimensions, allow. | |
| try: | |
| w=int(re.sub(r'\D','',str(im.get('width') or '0')) or 0);h=int(re.sub(r'\D','',str(im.get('height') or '0')) or 0) | |
| if (w and w<120) or (h and h<90):return -999 | |
| if w>=500 or h>=300:score+=3 | |
| except Exception:pass | |
| alt=(im.get('alt') or im.get('title') or '').lower() | |
| if any(x in alt for x in ['logo','avatar','quảng cáo','advertisement','banner']):return -999 | |
| cls=' '.join(im.get('class',[]));eid=im.get('id','') | |
| if BAD_RE.search(cls+' '+eid):return -999 | |
| if GOOD_RE.search(cls+' '+eid):score+=2 | |
| score+=_near_article_text_score(im) | |
| score+=_image_anchor_penalty(im,page_url) | |
| if any(x in low for x in ['cdn','photo','image','media','upload','thumb','avatar']):score+=1 | |
| # Tienphong and many VN papers use lazy/data src without figure; still accept if inside article block. | |
| if im.find_parent(['article','main']) or GOOD_RE.search(' '.join(block.get('class',[]))+' '+block.get('id','')):score+=3 | |
| return score | |
| def _extract_img_src(im, page_url): | |
| src=(im.get('data-src') or im.get('data-original') or im.get('data-lazy-src') or im.get('data-srcset') or im.get('srcset') or im.get('src') or '') | |
| if ',' in src:src=src.split(',')[0].strip().split(' ')[0] | |
| else:src=src.strip().split(' ')[0] | |
| return _abs_url(src,page_url) | |
| def _article_only_images(url): | |
| imgs=[] | |
| try: | |
| from bs4 import BeautifulSoup | |
| r=requests.get(url,headers=getattr(base,'HEADERS',{}),timeout=18);r.encoding='utf-8' | |
| soup=BeautifulSoup(r.text,'lxml') | |
| _clean_soup(soup) | |
| block=_find_article_block(soup) | |
| candidates=[] | |
| for el in block.find_all(['figure','picture'],recursive=True): | |
| im=el.find('img') | |
| if im and im not in candidates:candidates.append(im) | |
| for im in block.find_all('img',recursive=True): | |
| if im not in candidates:candidates.append(im) | |
| scored=[];seen=set() | |
| for im in candidates: | |
| src=_extract_img_src(im,url) | |
| if not src or src in seen:continue | |
| seen.add(src) | |
| sc=_image_score(im,src,block,url) | |
| if sc>=2: | |
| scored.append((sc,src)) | |
| # Keep original article order but only for scored images, filtering duplicate URLs. | |
| good=set(src for sc,src in sorted(scored,reverse=True) if sc>=2) | |
| for im in candidates: | |
| src=_extract_img_src(im,url) | |
| if src in good and src not in imgs:imgs.append(src) | |
| if len(imgs)>=20:break | |
| # Fallback: og:image is usually article main image, and better than no image. | |
| if not imgs: | |
| og=soup.find('meta',property='og:image') or soup.find('meta',attrs={'name':'twitter:image'}) | |
| if og: | |
| src=_abs_url(og.get('content',''),url) | |
| if src and not any(x in src.lower() for x in ['logo','icon','avatar','sprite']):imgs.append(src) | |
| except Exception:pass | |
| return imgs[:20] | |
| def _scrape_url_article_only(url): | |
| data=base.scrape_any_url(url) | |
| imgs=_article_only_images(url) | |
| data['images']=imgs | |
| data['image']=imgs[0] if imgs else '' | |
| return data | |
| # Override the functions used by inherited endpoints. | |
| f1._article_only_images=_article_only_images | |
| f1._scrape_url_article_only=_scrape_url_article_only | |
| # Replace URL endpoints to use improved extraction. | |
| _PATCH={('/api/url_wall','POST'),('/api/rewrite_share','POST'),('/api/ai/url','POST'),('/','GET')} | |
| app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)] | |
| async def final2_url_wall(request:Request): | |
| body=await request.json();url=base._clean_text(body.get('url','')) | |
| if not url.startswith('http'):return JSONResponse({'error':'missing url'},status_code=400) | |
| try:data=_scrape_url_article_only(url) | |
| except Exception as e:return JSONResponse({'error':'Không scrape được URL: '+str(e)[:180]},status_code=422) | |
| raw=(data.get('summary','')+'\n'+data.get('text','')).strip() | |
| if len(raw)<120:return JSONResponse({'error':'URL không có đủ nội dung để tóm tắt'},status_code=422) | |
| prompt=f"""Tóm tắt bài viết nguồn dưới đây để đăng lên Tường AI VNEWS. | |
| Yêu cầu: | |
| - Chỉ tóm tắt nội dung chính, không viết lại toàn bộ bài. | |
| - Ngắn gọn, cụ thể, dễ hiểu. | |
| - Không lặp ý, không thêm chi tiết ngoài nguồn. | |
| - Tối đa 5 ý chính hoặc 2 đoạn ngắn. | |
| - Hạn chế dùng dấu đầu dòng. | |
| Tiêu đề gốc: {data.get('title','')} | |
| Nguồn: {_domain(url)} | |
| Nội dung gốc: | |
| {raw[:16000]}""" | |
| text=await base.qwen_generate(prompt,image_url=(data.get('image') or None),max_tokens=900) | |
| if not text:text=rt.old._fallback_summary_from_prompt(prompt,max_units=5) if hasattr(rt.old,'_fallback_summary_from_prompt') else raw[:900] | |
| text=rt.postprocess(text) if hasattr(rt,'postprocess') else text | |
| src=[{'title':data.get('title'), 'url':url, 'excerpt':raw[:500], 'via':_domain(url)}] | |
| if 'Nguồn tham khảo:' not in text:text+='\n\n'+rt.source_line(src) | |
| imgs=data.get('images') or [] | |
| post=base.make_post(data.get('title') or 'Bài viết',text,imgs[0] if imgs else '',url,'url',sources=src) | |
| post['images']=imgs | |
| posts=base._load_ai_wall();posts.insert(0,post);base._save_ai_wall(posts) | |
| return JSONResponse({'post':post}) | |
| async def final2_rewrite_share(request:Request):return await final2_url_wall(request) | |
| async def final2_ai_url(request:Request):return await final2_url_wall(request) | |
| async def index_final2(): | |
| html=f1._load_index_html();body=getattr(rt.old,'PATCH_INJECT','') + f1.FINAL_INJECT | |
| return HTMLResponse(html.replace('</body>',body+'\n</body>') if '</body>' in html else html+body) | |