Restore ai_runtime_final.py from c93b544 - final runtime overrides with article-only images

ai_runtime_final.py

@@ -15,7 +15,6 @@ RESTORE_INDEX_URL = "https://huggingface.co/spaces/bep40/vnews/raw/restore-33c3d
 SPACE_URL = "https://bep40-vnews.hf.space"
 DEFAULT_IMG = "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
 
-# Only voices that support Vietnamese reliably. Extra labels map to these Vietnamese neural voices.
 VN_VOICES = {
     "nu": "vi-VN-HoaiMyNeural", "female": "vi-VN-HoaiMyNeural", "hoaimy": "vi-VN-HoaiMyNeural",
     "nu-tre": "vi-VN-HoaiMyNeural", "nu-truyen-cam": "vi-VN-HoaiMyNeural", "nu-tin-nhanh": "vi-VN-HoaiMyNeural",
@@ -35,7 +34,7 @@ def _domain(url):
 
 
 def _strip_bullet_prefix(s):
-    return clean(re.sub(r'^[\s•\-\*·▪▫●○\d\.\)\(]+', '', s or ''))
+    return clean(re.sub(r'^[\s\•\-\*·▪▫●○\d\.\)\(]+', '', s or ''))
 
 
 def _source_badge_url_first(post):
@@ -60,7 +59,6 @@ def _abs_url(src, base_url):
 
 def _article_content_block(soup):
     for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe']):tag.decompose()
-    # Aggressively remove related/ad/recommend containers before image collection.
     bad_re=re.compile(r'(related|relate|recommend|suggest|sidebar|ads|advert|popular|more|xem-them|xemthem|tin-lien-quan|tinlienquan|doc-them|docthem|other-news|news-other|article-related|box-tin|box_related|story-related|recommend-news|same-category|cate-list|news-list|most-view|banner|qc|quang-cao|sponsor)',re.I)
     for el in list(soup.find_all(True)):
         cls=' '.join(el.get('class',[])); eid=el.get('id',''); role=el.get('role','')
@@ -94,7 +92,6 @@ def _image_is_likely_article(im, src):
 
 
 def _article_only_images(url):
-    """Collect images only inside main article content. If uncertain, return fewer/no images rather than related/ad images."""
     imgs=[]
     try:
         from bs4 import BeautifulSoup
@@ -102,7 +99,6 @@ def _article_only_images(url):
         soup=BeautifulSoup(r.text,'lxml')
         block=_article_content_block(soup)
         candidates=[]
-        # Prefer figure/picture under article body; then direct img in body.
         for el in block.find_all(['figure','picture'],recursive=True):
             im=el.find('img')
             if im:candidates.append(im)
@@ -115,12 +111,10 @@ def _article_only_images(url):
             else:src=src.strip().split(' ')[0]
             src=_abs_url(src,url)
             if src in seen or not _image_is_likely_article(im,src):continue
-            # parent text guard: skip images from any remaining related block
             parent_txt=' '.join((im.parent.get('class',[]) if im.parent else []))+' '+(im.parent.get('id','') if im.parent else '')
             if re.search(r'(related|recommend|tin-lien-quan|doc-them|xem-them|popular|ads|banner)',parent_txt,re.I):continue
             seen.add(src);imgs.append(src)
             if len(imgs)>=20:break
-        # Use og:image ONLY as article main image fallback when no body image found.
         if not imgs:
             og=soup.find('meta',property='og:image') or soup.find('meta',attrs={'name':'twitter:image'})
             if og:
@@ -154,7 +148,6 @@ def _download_image_safe(url, fallback_title, out_path):
             r=requests.get(url,headers=getattr(base,'HEADERS',{}),timeout=18)
             if r.status_code==200 and len(r.content)>1200:
                 with open(out_path,'wb') as f:f.write(r.content)
-                # verify PIL opens it
                 if Image:
                     Image.open(out_path).verify()
                 return out_path
@@ -167,7 +160,6 @@ def _download_image_safe(url, fallback_title, out_path):
 
 def final_make_tts(text,voice,out_path):
     text=_strip_bullet_prefix(text) or 'Bản tin VNEWS.'
-    # Only Vietnamese voices. Unknown choices fall back to Vietnamese female.
     edge_voice=VN_VOICES.get(str(voice or '').lower().strip(), 'vi-VN-HoaiMyNeural')
     for ev in [edge_voice, 'vi-VN-HoaiMyNeural', 'vi-VN-NamMinhNeural']:
         try:
@@ -178,7 +170,6 @@ def final_make_tts(text,voice,out_path):
         base.gTTS(text,lang='vi',tld='com.vn',slow=False).save(out_path)
         if os.path.exists(out_path) and os.path.getsize(out_path)>1000:return out_path
     except Exception:pass
-    # Last-resort silent audio guarantees short generation succeeds.
     subprocess.run(['ffmpeg','-y','-f','lavfi','-i','anullsrc=channel_layout=stereo:sample_rate=44100','-t','3','-q:a','9','-acodec','libmp3lame',out_path],stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30)
     return out_path
 
@@ -219,138 +210,7 @@ def final_make_frame(post,seg,idx,total,img_path,out_path):
     title_lines=rt.wrap_text(draw,_strip_bullet_prefix(post.get('title','')),fs,W-120,3);y2=1640;draw.line((80,y2-26,W-80,y2-26),fill=(70,70,70),width=2);_draw_center(draw,title_lines,fs,y2,(220,220,220),W,42)
     bg.save(out_path,quality=92)
 
-# Monkey patches for old functions.
 rt.make_frame=final_make_frame;rt.make_tts=final_make_tts;rt._source_badge=_source_badge_url_first
 
-# Override endpoints.
 _PATCH={('/api/url_wall','POST'),('/api/rewrite_share','POST'),('/api/ai/url','POST'),('/api/ai/short/{post_id}','POST'),('/','GET'),('/aw','GET')}
 app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
-
-@app.post('/api/url_wall')
-async def final_url_wall(request:Request):
-    body=await request.json();url=base._clean_text(body.get('url',''))
-    if not url.startswith('http'):return JSONResponse({'error':'missing url'},status_code=400)
-    try:data=_scrape_url_article_only(url)
-    except Exception as e:return JSONResponse({'error':'Không scrape được URL: '+str(e)[:180]},status_code=422)
-    raw=(data.get('summary','')+'\n'+data.get('text','')).strip()
-    if len(raw)<120:return JSONResponse({'error':'URL không có đủ nội dung để tóm tắt'},status_code=422)
-    prompt=f"""Tóm tắt bài viết nguồn dưới đây để đăng lên Tường AI VNEWS.
-
-Yêu cầu:
-- Chỉ tóm tắt nội dung chính, không viết lại toàn bộ bài.
-- Ngắn gọn, cụ thể, dễ hiểu.
-- Không lặp ý, không thêm chi tiết ngoài nguồn.
-- Tối đa 5 ý chính hoặc 2 đoạn ngắn.
-- Hạn chế dùng dấu đầu dòng.
-
-Tiêu đề gốc: {data.get('title','')}
-Nguồn: {_domain(url)}
-Nội dung gốc:
-{raw[:16000]}"""
-    text=await base.qwen_generate(prompt,image_url=(data.get('image') or None),max_tokens=900)
-    if not text:text=rt.old._fallback_summary_from_prompt(prompt,max_units=5) if hasattr(rt.old,'_fallback_summary_from_prompt') else raw[:900]
-    text=rt.postprocess(text) if hasattr(rt,'postprocess') else text
-    src=[{'title':data.get('title'), 'url':url, 'excerpt':raw[:500], 'via':_domain(url)}]
-    if 'Nguồn tham khảo:' not in text:text+='\n\n'+rt.source_line(src)
-    imgs=data.get('images') or []
-    post=base.make_post(data.get('title') or 'Bài viết',text,imgs[0] if imgs else '',url,'url',sources=src)
-    post['images']=imgs
-    posts=base._load_ai_wall();posts.insert(0,post);base._save_ai_wall(posts)
-    return JSONResponse({'post':post})
-
-@app.post('/api/rewrite_share')
-async def final_rewrite_share(request:Request):return await final_url_wall(request)
-@app.post('/api/ai/url')
-async def final_ai_url(request:Request):return await final_url_wall(request)
-
-@app.post('/api/ai/short/{post_id}')
-async def final_short(post_id:str,request:Request):
-    try:body=await request.json()
-    except Exception:body={}
-    voice=str(body.get('voice','nu')).lower().strip();emotion=str(body.get('emotion','neutral')).lower().strip();speed=max(0.85,min(1.35,float(body.get('speed',1.2) or 1.2)))
-    posts=base._load_ai_wall();post=next((p for p in posts if str(p.get('id'))==str(post_id)),None)
-    if not post:return JSONResponse({'error':'post not found'},status_code=404)
-    segs=rt.split_segments(post,8) if hasattr(rt,'split_segments') else [_strip_bullet_prefix(post.get('text') or post.get('title') or 'VNEWS')]
-    imgs=[u for u in (post.get('images') or []) if u] or ([post.get('img')] if post.get('img') else [])
-    os.makedirs(base.SHORTS_DIR,exist_ok=True);suffix=f'_{voice}_{emotion}_{str(speed).replace(".","p")}_articleimgs_vivoice'
-    out=os.path.join(base.SHORTS_DIR,base._safe_name(post_id+suffix)+'.mp4')
-    if os.path.exists(out):
-        post['video']='/api/ai/short-file/'+post_id+suffix;base._save_ai_wall(posts);return JSONResponse({'video':post['video'],'segments':len(segs),'subtitles':False})
-    work=os.path.join(base.SHORTS_DIR,base._safe_name(post_id+suffix));os.makedirs(work,exist_ok=True)
-    clips=[]
-    try:
-        for i,seg in enumerate(segs):
-            img_url=imgs[i % len(imgs)] if imgs else ''
-            img=os.path.join(work,f'image_{i}.jpg');frame=os.path.join(work,f'f{i}.jpg');aud=os.path.join(work,f'a{i}.mp3');aud2=os.path.join(work,f'a{i}_fast.mp3');clip=os.path.join(work,f'c{i}.mp4')
-            _download_image_safe(img_url,post.get('title','AI news'),img)
-            seg=_strip_bullet_prefix(seg);final_make_frame(post,seg,i,len(segs),img,frame)
-            prefix={'urgent':'Tin nhanh.','warm':'Câu chuyện đáng chú ý.','serious':'Bản tin nghiêm túc.','energetic':'Cập nhật nổi bật.'}.get(emotion,'')
-            spoken=(prefix+' '+seg).strip() if i==0 and prefix else seg
-            final_make_tts(spoken,voice,aud)
-            try:subprocess.run(['ffmpeg','-y','-i',aud,'-filter:a',f'atempo={speed}','-vn',aud2],check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=120)
-            except Exception:aud2=aud
-            try:
-                subprocess.run(['ffmpeg','-y','-loop','1','-i',frame,'-i',aud2,'-shortest','-c:v','libx264','-tune','stillimage','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k','-vf','scale=1080:1920',clip],check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=180)
-            except Exception:
-                # last-resort visual-only 4s clip
-                subprocess.run(['ffmpeg','-y','-loop','1','-t','4','-i',frame,'-f','lavfi','-i','anullsrc=channel_layout=stereo:sample_rate=44100','-shortest','-c:v','libx264','-pix_fmt','yuv420p','-c:a','aac','-vf','scale=1080:1920',clip],check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=120)
-            clips.append(clip)
-        lf=os.path.join(work,'list.txt')
-        with open(lf,'w',encoding='utf-8') as f:
-            for c in clips:f.write("file '"+c.replace("'","'\\''")+"'\n")
-        subprocess.run(['ffmpeg','-y','-f','concat','-safe','0','-i',lf,'-c','copy',out],check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=240)
-        post['video']='/api/ai/short-file/'+post_id+suffix;post['short_subtitles']=False;post['short_segments']=segs;post['short_speed']=speed;base._save_ai_wall(posts)
-        return JSONResponse({'video':post['video'],'segments':len(segs),'subtitles':False})
-    except Exception as e:return JSONResponse({'error':'Không tạo được shorts: '+str(e)[:220]},status_code=500)
-
-@app.get('/aw')
-def ai_wall_share(post:str=Query(default=''), short:int=Query(default=0)):
-    posts=base._load_ai_wall();p=next((x for x in posts if str(x.get('id'))==str(post)),None)
-    if not p:return HTMLResponse(f'<script>location.href="{SPACE_URL}"</script>')
-    title=p.get('title') or 'VNEWS AI';img=p.get('img') or DEFAULT_IMG
-    desc=(p.get('text') or '')[:220]
-    return HTMLResponse(f'''<!doctype html><html><head><meta charset="utf-8"><title>{title}</title><meta property="og:title" content="{title}"><meta property="og:description" content="{desc}"><meta property="og:image" content="{img}"><meta property="og:type" content="article"><meta name="twitter:card" content="summary_large_image"></head><body><script>localStorage.setItem('pending_ai_post','{post}');location.href='{SPACE_URL}'</script></body></html>''')
-
-FINAL_INJECT = r'''
-<style>
-#ai-topic-input{display:none!important}button[onclick*="createTopicPost"],*[onclick*="createTopicPost"]{display:none!important}.ai-topic-row,.topic-row,.ai-compose-topic{display:none!important}.ai-compose{width:calc(100% - 8px)!important}.ai-compose-row:has(#ai-url-input){display:flex!important;flex-direction:column!important;gap:8px!important;width:100%!important}.ai-compose-row:has(#ai-url-input) button{width:100%!important;display:block!important}.ai-compose-row:has(#ai-url-input) input,#ai-url-input{display:block!important;width:100%!important;max-width:none!important;flex:1 1 100%!important;box-sizing:border-box!important}.ai-url-only-note{font-size:11px;color:#888;margin:5px 0 8px;width:100%}.ai-wall-gallery{display:grid;grid-template-columns:repeat(2,1fr);gap:6px;margin:10px 0}.ai-wall-gallery img{width:100%;aspect-ratio:16/9;object-fit:cover;border-radius:8px;background:#222}.ai-wall-gallery img:first-child{grid-column:1/-1}.ai-wall-final{margin:6px 4px;background:#1a1a1a;border:1px solid #2a2a2a;border-radius:8px;overflow:hidden}
-</style>
-<script>
-(function(){
-function esc(s){return String(s||'').replace(/[&<>"']/g,m=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[m]));}
-let finalWall=[];
-function hideTopicControls(){document.querySelectorAll('#ai-topic-input').forEach(e=>{let row=e.closest('.ai-compose-topic,.topic-row,.ai-topic-row')||e.parentElement;if(row&&!row.querySelector('#ai-url-input'))row.style.display='none';else e.style.display='none';});document.querySelectorAll('button,a').forEach(b=>{let t=(b.textContent||'').toLowerCase();let oc=b.getAttribute('onclick')||'';if(oc.includes('createTopicPost')||t.includes('chủ đề'))b.style.display='none';});let url=document.getElementById('ai-url-input');if(url&&!document.getElementById('ai-url-only-note')){let n=document.createElement('div');n.id='ai-url-only-note';n.className='ai-url-only-note';n.textContent='Dán URL bài viết; ảnh chỉ lấy từ nội dung bài, không lấy bài liên quan/quảng cáo.';url.insertAdjacentElement('afterend',n);}}
-function voiceOptions(){return `<option value="nu">Nữ Việt - Hoài My</option><option value="nam">Nam Việt - Nam Minh</option><option value="nu-truyen-cam">Nữ Việt truyền cảm</option><option value="nu-tin-nhanh">Nữ Việt tin nhanh</option><option value="nam-tram">Nam Việt trầm</option><option value="nam-ban-tin">Nam Việt bản tin</option><option value="nam-nang-dong">Nam Việt năng động</option>`}
-function addVoiceOptions(){document.querySelectorAll('#ai-short-voice, select[id*=voice]').forEach(sel=>{let old=sel.value||'nu';sel.innerHTML=voiceOptions();sel.value=[...sel.options].some(o=>o.value===old)?old:'nu';});}
-function shareAI(p,isShort){let url=location.origin+'/aw?post='+encodeURIComponent(p.id)+(isShort?'&short=1':'');let title=(isShort?'🎬 Short AI: ':'🧱 Tường AI: ')+(p.title||'VNEWS');if(navigator.share)navigator.share({title,url}).catch(()=>{});else navigator.clipboard.writeText(url).then(()=>alert('Đã sao chép link chia sẻ!')).catch(()=>{});}
-function galleryHtml(p){let imgs=(p.images||[]).filter(Boolean);if(!imgs.length&&p.img)imgs=[p.img];if(!imgs.length)return '';return '<div class="ai-wall-gallery">'+imgs.slice(0,12).map(u=>`<img src="${esc(u)}" loading="lazy">`).join('')+'</div>';}
-async function loadWall(){try{finalWall=(await (await fetch('/api/ai_wall')).json()).posts||[];renderWall();let pend=localStorage.getItem('pending_ai_post');if(pend){localStorage.removeItem('pending_ai_post');let idx=finalWall.findIndex(p=>String(p.id)===String(pend));if(idx>=0)setTimeout(()=>readFinalWall(idx),300);}}catch(e){}}
-function renderWall(){let home=document.getElementById('view-home');if(!home||!finalWall.length)return;document.getElementById('ai-wall-final')?.remove();let wrap=document.createElement('div');wrap.id='ai-wall-final';wrap.className='ai-wall-final';let h='<div class="slider-header"><span class="slider-label">🧱 Tường AI</span><span class="slider-note">URL summary</span></div><div class="slider-track">';finalWall.slice(0,30).forEach((p,i)=>{h+=`<div class="wall-item"><div class="wall-thumb">${p.img?`<img src="${esc(p.img)}">`:''}</div><div class="wall-title">${esc(p.title)}</div><div class="wall-text">${esc(p.text)}</div><div class="wall-actions"><button class="primary" onclick="readFinalWall(${i})">Xem</button><button onclick="shareFinalWall(${i})">📤 Chia sẻ</button>${p.video?`<button onclick="readFinalShort(${i})">Short</button>`:''}</div></div>`});h+='</div>';wrap.innerHTML=h;let compose=document.querySelector('.ai-compose');if(compose)compose.after(wrap);else home.prepend(wrap);}
-window.shareFinalWall=function(i){let p=finalWall[i];if(p)shareAI(p,false)};
-window.shareFinalShort=function(i){let p=finalWall[i];if(p)shareAI(p,true)};
-window.readFinalWall=window.aiReadWallPatched=window.aiReadWall=function(i){let p=finalWall[i]||(window.patchedWall||window.aiWall||[])[i];if(!p)return;showView('view-article');let sources='';if(p.sources&&p.sources.length){sources='<div class="article-summary"><b>Nguồn tham khảo:</b><br>'+p.sources.slice(0,5).map(s=>`• ${esc(s.title||s.url||'Nguồn')} ${s.url?`(${esc(new URL(s.url).hostname.replace('www.',''))})`:''}`).join('<br>')+'</div>'}let voiceBox=`<div class="article-actions"><select id="ai-short-voice">${voiceOptions()}</select><select id="ai-short-emotion"><option value="neutral">Trung tính</option><option value="urgent">Tin nhanh</option><option value="warm">Ấm áp</option><option value="serious">Nghiêm túc</option><option value="energetic">Sôi nổi</option></select><button onclick="makeFinalShort(${i})">🎬 Tạo short</button><button onclick="shareFinalWall(${i})">📤 Chia sẻ bài</button>${p.video?`<button onclick="shareFinalShort(${i})">📤 Chia sẻ short</button>`:''}</div>`;let h=`<button class="back-btn" onclick="switchCat('home')">← Quay lại</button><div class="article-view"><span class="badge badge-ai">AI</span><h1 class="article-title">${esc(p.title)}</h1>${galleryHtml(p)}${sources}<p class="article-p" style="white-space:pre-wrap">${esc(p.text)}</p>${p.video?`<video class="article-img" src="${p.video}" controls playsinline></video>`:''}<div class="article-actions">${p.url?`<button onclick="window.open('${p.url}','_blank')">🔗 Nguồn</button>`:''}</div>${voiceBox}</div>`;document.getElementById('view-article').innerHTML=h;addVoiceOptions();window.scrollTo(0,0)};
-window.readFinalShort=function(i){let p=finalWall[i];if(!p||!p.video)return;showView('view-article');document.getElementById('view-article').innerHTML=`<button class="back-btn" onclick="switchCat('home')">← Quay lại</button><div class="article-view"><span class="badge badge-ai">Short AI</span><h1 class="article-title">${esc(p.title)}</h1><video class="article-img" src="${p.video}" controls playsinline autoplay></video><div class="article-actions"><button onclick="shareFinalShort(${i})">📤 Chia sẻ short</button><button onclick="readFinalWall(${i})">Xem bài tường</button></div></div>`;window.scrollTo(0,0)};
-window.makeFinalShort=window.aiMakeShortPatched=async function(i){let p=finalWall[i]||(window.patchedWall||window.aiWall||[])[i];if(!p)return;let voice=document.getElementById('ai-short-voice')?.value||'nu';let emotion=document.getElementById('ai-short-emotion')?.value||'neutral';let btn=event?.target;if(btn){btn.disabled=true;btn.textContent='Đang tạo...'}try{let r=await fetch('/api/ai/short/'+p.id,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({voice,emotion,speed:1.2})});let j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi tạo short');p.video=j.video;let idx=finalWall.findIndex(x=>x.id===p.id);if(idx<0){finalWall.unshift(p);idx=0}renderWall();readFinalWall(idx);alert('Đã tạo short AI và cập nhật ngay trên tường.');}catch(e){alert('Không tạo được short: '+e.message)}finally{if(btn){btn.disabled=false;btn.textContent='🎬 Tạo short'}}};
-window.createTopicPost=function(){alert('Đã tắt ô nhập chủ đề. Vui lòng dán URL bài viết.');};
-window.createUrlPost=async function(){let inp=document.getElementById('ai-url-input');let url=(inp&&inp.value||'').trim();if(!url)return alert('Dán URL trước');if(!/^https?:\/\//i.test(url))return alert('URL cần bắt đầu bằng http:// hoặc https://');let btn=event?.target;if(btn){btn.disabled=true;btn.textContent='Đang tóm tắt...'}try{let r=await fetch('/api/url_wall',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})});let j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi URL');finalWall.unshift(j.post);if(inp)inp.value='';renderWall();readFinalWall(0);alert('Đã đăng tường AI, không cần tải lại trang.');}catch(e){alert(e.message||'Lỗi URL')}finally{if(btn){btn.disabled=false;btn.textContent='Chèn URL'}}};
-setTimeout(()=>{hideTopicControls();addVoiceOptions();loadWall();},300);setInterval(()=>{hideTopicControls();addVoiceOptions();},1200);
-})();
-</script>
-'''
-
-def _load_index_html():
-    try:
-        with open('/app/static/index.html','r',encoding='utf-8') as f:html=f.read()
-    except Exception:html=''
-    if '<!DOCTYPE html>' not in html or '<div id="view-home"' not in html:
-        try:
-            r=requests.get(RESTORE_INDEX_URL,timeout=20)
-            if r.status_code==200 and '<!DOCTYPE html>' in r.text:html=r.text
-        except Exception:pass
-    if '<!DOCTYPE html>' not in html:html='<!DOCTYPE html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>VNEWS</title></head><body><div id="view-home">VNEWS</div></body></html>'
-    return html
-
-@app.get('/')
-async def index_final():
-    html=_load_index_html();body=getattr(rt.old,'PATCH_INJECT','') + FINAL_INJECT
-    return HTMLResponse(html.replace('</body>',body+'\n</body>') if '</body>' in html else html+body)