Spaces:
Running
Running
Add hot Google keywords under topic box + improve topic research guidance
Browse files- ai_runtime_final6.py +81 -85
ai_runtime_final6.py
CHANGED
|
@@ -1,25 +1,24 @@
|
|
| 1 |
-
"""Final6: robust topic synthesis
|
| 2 |
-
import re, time, json, os, threading, html as html_lib
|
| 3 |
from urllib.parse import quote, urlparse, parse_qs, unquote
|
| 4 |
import requests
|
| 5 |
from bs4 import BeautifulSoup
|
| 6 |
import ai_runtime_final5 as f5
|
| 7 |
from ai_runtime_final5 import app, rt, HTMLResponse, JSONResponse, Request, Query
|
| 8 |
|
| 9 |
-
|
| 10 |
-
_PATCH={('/api/topic_post','POST'),('/api/shorts','GET'),('/','GET')}
|
| 11 |
app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
|
| 12 |
|
| 13 |
_TOPIC_CACHE={}
|
|
|
|
| 14 |
_SHORTS_CACHE_FINAL6={"t":0,"d":[]}
|
| 15 |
_TRANSLATE_CACHE_PATH="/data/title_vi_cache.json" if os.path.isdir('/data') else "/app/data/title_vi_cache.json"
|
| 16 |
_translate_lock=threading.Lock()
|
| 17 |
YOUTUBE_HANDLES=["baodantri7941","baosuckhoedoisongboyte"]
|
| 18 |
UA={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Accept-Language":"vi,en;q=0.8"}
|
|
|
|
| 19 |
|
| 20 |
-
def clean(s):
|
| 21 |
-
return re.sub(r"\s+"," ",html_lib.unescape(str(s or ""))).strip()
|
| 22 |
-
|
| 23 |
def _domain(u):
|
| 24 |
try:return urlparse(u or '').netloc.replace('www.','')
|
| 25 |
except Exception:return ''
|
|
@@ -30,11 +29,9 @@ def _load_title_cache():
|
|
| 30 |
with open(_TRANSLATE_CACHE_PATH,'r',encoding='utf-8') as f:return json.load(f)
|
| 31 |
except Exception:pass
|
| 32 |
return {}
|
| 33 |
-
|
| 34 |
def _save_title_cache(db):
|
| 35 |
try:
|
| 36 |
-
os.makedirs(os.path.dirname(_TRANSLATE_CACHE_PATH),exist_ok=True)
|
| 37 |
-
tmp=_TRANSLATE_CACHE_PATH+'.tmp'
|
| 38 |
with open(tmp,'w',encoding='utf-8') as f:json.dump(db,f,ensure_ascii=False)
|
| 39 |
os.replace(tmp,_TRANSLATE_CACHE_PATH)
|
| 40 |
except Exception:pass
|
|
@@ -42,14 +39,11 @@ def _save_title_cache(db):
|
|
| 42 |
def _looks_vietnamese(s):
|
| 43 |
s=s or ''
|
| 44 |
if re.search(r'[àáạảãâầấậẩẫăằắặẳẵèéẹẻẽêềếệểễìíịỉĩòóọỏõôồốộổỗơờớợởỡùúụủũưừứựửữỳýỵỷỹđ]',s,re.I):return True
|
| 45 |
-
vi_words=[' và ',' của ',' người ',' tại ',' trong ',' với ',' không ',' được ',' sau ',' trước ',' công an ',' bệnh viện ',' học sinh ',' tài xế ',' bóng đá ',' tin tức ',' sức khỏe ']
|
| 46 |
low=' '+s.lower()+' '
|
| 47 |
-
return any(w in low for w in
|
| 48 |
-
|
| 49 |
def _translate_title_vi(title):
|
| 50 |
title=clean(title)
|
| 51 |
-
if not title:return title
|
| 52 |
-
if _looks_vietnamese(title):return title
|
| 53 |
with _translate_lock:
|
| 54 |
db=_load_title_cache()
|
| 55 |
if title in db:return db[title]
|
|
@@ -57,15 +51,61 @@ def _translate_title_vi(title):
|
|
| 57 |
try:
|
| 58 |
r=requests.get('https://translate.googleapis.com/translate_a/single',params={'client':'gtx','sl':'auto','tl':'vi','dt':'t','q':title},headers=UA,timeout=8)
|
| 59 |
if r.status_code==200:
|
| 60 |
-
data=r.json()
|
| 61 |
-
vi=''.join(part[0] for part in data[0] if part and part[0]).strip() or title
|
| 62 |
except Exception:pass
|
| 63 |
vi=clean(vi)
|
| 64 |
with _translate_lock:
|
| 65 |
db=_load_title_cache();db[title]=vi;_save_title_cache(db)
|
| 66 |
return vi
|
| 67 |
|
| 68 |
-
# =====
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
def _unwrap_ddg_href(href):
|
| 70 |
if not href:return ''
|
| 71 |
if href.startswith('//duckduckgo.com/l/?') or 'duckduckgo.com/l/?' in href:
|
|
@@ -82,9 +122,7 @@ def _ddg_search(topic, limit=10):
|
|
| 82 |
for res in soup.select('.result'):
|
| 83 |
a=res.select_one('.result__title a') or res.find('a',href=True)
|
| 84 |
if not a:continue
|
| 85 |
-
link=_unwrap_ddg_href(a.get('href',''))
|
| 86 |
-
title=clean(a.get_text(' ',strip=True))
|
| 87 |
-
snippet=clean((res.select_one('.result__snippet') or res).get_text(' ',strip=True))
|
| 88 |
if not link.startswith('http') or link in seen:continue
|
| 89 |
if any(bad in link for bad in ['duckduckgo.com','youtube.com','facebook.com','tiktok.com']):continue
|
| 90 |
seen.add(link);items.append({'title':title,'url':link,'source':_domain(link),'snippet':snippet})
|
|
@@ -112,8 +150,7 @@ def _extract_article_text_bs(url, max_chars=6500):
|
|
| 112 |
try:
|
| 113 |
r=requests.get(url,headers=UA,timeout=16,allow_redirects=True)
|
| 114 |
if r.status_code>=400:return ''
|
| 115 |
-
r.encoding='utf-8'
|
| 116 |
-
soup=BeautifulSoup(r.text,'lxml')
|
| 117 |
for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe','svg']):tag.decompose()
|
| 118 |
candidates=[]
|
| 119 |
for sel in ['article','main','.article-content','.detail-content','.singular-content','.fck_detail','.content-detail','.entry-content','.story-body','.knc-content']:
|
|
@@ -124,8 +161,7 @@ def _extract_article_text_bs(url, max_chars=6500):
|
|
| 124 |
ps=[]
|
| 125 |
for el in best.find_all(['p','h2','h3'],recursive=True):
|
| 126 |
t=clean(el.get_text(' ',strip=True))
|
| 127 |
-
if len(t)>45 and not any(x in t.lower() for x in ['đăng ký nhận tin','theo dõi chúng tôi','chuyên mục','xem thêm','tin liên quan']):
|
| 128 |
-
ps.append(t)
|
| 129 |
if sum(len(x) for x in ps)>max_chars:break
|
| 130 |
return '\n'.join(ps)[:max_chars]
|
| 131 |
except Exception:return ''
|
|
@@ -152,13 +188,11 @@ def _scrape_article_text(url, max_chars=6500):
|
|
| 152 |
def _web_research_context(topic):
|
| 153 |
now=time.time();key=topic.lower().strip()
|
| 154 |
if key in _TOPIC_CACHE and now-_TOPIC_CACHE[key]['t']<900:return _TOPIC_CACHE[key]['d']
|
| 155 |
-
# Direct search first; RSS only supplements snippets/titles.
|
| 156 |
items=_ddg_search(topic,10)
|
| 157 |
if len(items)<4:
|
| 158 |
seen={i['url'] for i in items}
|
| 159 |
for it in _google_news_items(topic,8):
|
| 160 |
-
if it['url'] not in seen:
|
| 161 |
-
items.append(it);seen.add(it['url'])
|
| 162 |
blocks=[];sources=[]
|
| 163 |
for it in items[:10]:
|
| 164 |
text=_scrape_article_text(it['url'],6500)
|
|
@@ -181,10 +215,7 @@ def _topic_image(topic):
|
|
| 181 |
async def topic_post_synthesis(request:Request):
|
| 182 |
body=await request.json();topic=clean(body.get('topic',''))
|
| 183 |
if not topic:return JSONResponse({'error':'missing topic'},status_code=400)
|
| 184 |
-
img=_topic_image(topic)
|
| 185 |
-
research=_web_research_context(topic)
|
| 186 |
-
context=research.get('context','')
|
| 187 |
-
sources=research.get('sources',[])
|
| 188 |
prompt=f"""Bạn là biên tập viên VNEWS. Người dùng chọn chủ đề: "{topic}".
|
| 189 |
|
| 190 |
Dưới đây là NỘI DUNG các bài viết/đoạn mô tả đã crawl từ internet. Hãy đọc hiểu và TỔNG HỢP thành MỘT BÀI VIẾT HOÀN CHỈNH. Tuyệt đối không bê nguyên văn, không xếp danh sách tiêu đề thành bài viết, không viết kiểu trả lời chat.
|
|
@@ -208,37 +239,29 @@ Yêu cầu bắt buộc:
|
|
| 208 |
body=block.split('NỘI DUNG BÀI VIẾT ĐÃ CRAWL:')[-1].split('ĐOẠN MÔ TẢ TỪ KẾT QUẢ TÌM KIẾM:')[-1].strip()
|
| 209 |
if len(body)>120:parts.append(body)
|
| 210 |
joined='\n\n'.join(parts)[:7000]
|
| 211 |
-
text=(f"{topic}: những điểm chính cần biết\n\n"
|
| 212 |
-
|
| 213 |
-
+ (joined if joined else 'Hiện dữ liệu crawl còn hạn chế, bài viết này chỉ đưa ra bối cảnh tổng quan và các điểm cần theo dõi tiếp.') +
|
| 214 |
-
"\n\nNguồn tham khảo: " + ', '.join(sorted({s.get('via','') for s in sources if s.get('via')})) )
|
| 215 |
-
post=f5.base.make_post(topic,text,img,'','topic_web_synthesis',sources=[s for s in sources if s.get('url')])
|
| 216 |
-
post['images']=[img]
|
| 217 |
posts=f5.base._load_ai_wall();posts.insert(0,post);f5.base._save_ai_wall(posts)
|
| 218 |
return JSONResponse({'post':post})
|
| 219 |
|
| 220 |
-
# ===== Stable newest
|
| 221 |
def _yt_ytdlp(handle,count=30):
|
| 222 |
try:
|
| 223 |
import yt_dlp
|
| 224 |
urls=[f'https://www.youtube.com/@{handle}/shorts',f'https://www.youtube.com/@{handle}/videos']
|
| 225 |
-
out=[];seen=set()
|
| 226 |
-
opts={'quiet':True,'extract_flat':True,'skip_download':True,'playlistend':count,'ignoreerrors':True,'no_warnings':True,'extractor_args':{'youtube':{'player_client':['web']}}}
|
| 227 |
for url in urls:
|
| 228 |
-
with yt_dlp.YoutubeDL(opts) as ydl:
|
| 229 |
-
info=ydl.extract_info(url,download=False)
|
| 230 |
for e in (info or {}).get('entries') or []:
|
| 231 |
vid=e.get('id') or ''
|
| 232 |
if not re.match(r'^[A-Za-z0-9_-]{11}$',vid) or vid in seen:continue
|
| 233 |
title=e.get('title') or 'YouTube Short'
|
| 234 |
-
if url.endswith('/videos') and '#short' not in title.lower() and 'shorts' not in title.lower():
|
| 235 |
-
continue
|
| 236 |
seen.add(vid);out.append({'title':title,'link':'https://www.youtube.com/watch?v='+vid,'img':'https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg','source':'yt','id':vid,'channel':handle})
|
| 237 |
if len(out)>=count:break
|
| 238 |
if len(out)>=count:break
|
| 239 |
return out
|
| 240 |
except Exception:return []
|
| 241 |
-
|
| 242 |
def _yt_html(handle,count=30):
|
| 243 |
out=[];seen=set()
|
| 244 |
for suffix in ['shorts','videos']:
|
|
@@ -247,8 +270,7 @@ def _yt_html(handle,count=30):
|
|
| 247 |
for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
|
| 248 |
vid=m.group(1)
|
| 249 |
if vid in seen:continue
|
| 250 |
-
snip=html[max(0,m.start()-1200):m.start()+2200]
|
| 251 |
-
title='YouTube Short'
|
| 252 |
mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip) or re.search(r'"accessibilityText":"([^"]+)"',snip)
|
| 253 |
if mt:title=clean(mt.group(1).replace('\\n',' '))
|
| 254 |
if suffix=='videos' and '#short' not in title.lower() and 'shorts' not in title.lower():continue
|
|
@@ -257,70 +279,44 @@ def _yt_html(handle,count=30):
|
|
| 257 |
except Exception:pass
|
| 258 |
if len(out)>=count:break
|
| 259 |
return out[:count]
|
| 260 |
-
|
| 261 |
def _fallback_shorts():
|
| 262 |
try:return f5._fallback_shorts()
|
| 263 |
except Exception:return []
|
| 264 |
-
|
| 265 |
@app.get('/api/shorts')
|
| 266 |
def api_shorts_final6(refresh:int=Query(default=0)):
|
| 267 |
now=time.time()
|
| 268 |
-
if not refresh and _SHORTS_CACHE_FINAL6['d'] and now-_SHORTS_CACHE_FINAL6['t']<600:
|
| 269 |
-
return JSONResponse(_SHORTS_CACHE_FINAL6['d'])
|
| 270 |
raw=[]
|
| 271 |
-
for h in YOUTUBE_HANDLES:
|
| 272 |
-
got=_yt_ytdlp(h,30) or _yt_html(h,30)
|
| 273 |
-
raw.extend(got)
|
| 274 |
raw.extend(_fallback_shorts())
|
| 275 |
seen=set();out=[]
|
| 276 |
for v in raw:
|
| 277 |
vid=v.get('id') or ''
|
| 278 |
if not vid:
|
| 279 |
-
m=re.search(r'(?:v=|shorts/|youtu\.be/)([A-Za-z0-9_-]{11})',v.get('link',''))
|
| 280 |
-
|
| 281 |
-
title=_translate_title_vi(v.get('title') or 'YouTube Short')
|
| 282 |
-
key=vid or re.sub(r'\W+','',title.lower())[:80]
|
| 283 |
if not key or key in seen:continue
|
| 284 |
-
seen.add(key)
|
| 285 |
-
item
|
| 286 |
-
|
| 287 |
-
item['link']='https://www.youtube.com/watch?v='+vid
|
| 288 |
-
item['img']='https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg'
|
| 289 |
-
item['source']='yt'
|
| 290 |
-
out.append(item)
|
| 291 |
if len(out)>=40:break
|
| 292 |
_SHORTS_CACHE_FINAL6.update({'t':now,'d':out})
|
| 293 |
return JSONResponse(out)
|
| 294 |
|
| 295 |
FINAL6_INJECT=r'''
|
| 296 |
<style>
|
| 297 |
-
#ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4{display:none!important}.topic-final5{display:flex!important}.ai-wall-topic-live{margin:6px 4px;background:#1a1a1a;border:1px solid #2a2a2a;border-radius:8px;overflow:hidden}
|
| 298 |
</style>
|
| 299 |
<script>
|
| 300 |
(function(){
|
| 301 |
function esc(s){return String(s||'').replace(/[&<>"']/g,m=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[m]));}
|
| 302 |
let liveTopicWall=[];
|
| 303 |
-
async function
|
| 304 |
-
|
| 305 |
-
let labels=[...document.querySelectorAll('.slider-wrap .slider-label')];
|
| 306 |
-
let wraps=labels.filter(l=>/shorts|short /i.test(l.textContent||'')&&!/short ai/i.test(l.textContent||'')).map(l=>l.closest('.slider-wrap')).filter(Boolean);
|
| 307 |
-
wraps.forEach((w,i)=>{if(i>0)w.remove();});
|
| 308 |
-
let w=wraps[0];
|
| 309 |
-
if(w){
|
| 310 |
-
let seen=new Set();[...w.querySelectorAll('.slider-item')].forEach(it=>{let img=it.querySelector('img')?.src||'';let tt=(it.querySelector('.slider-title')?.textContent||'').trim().toLowerCase();let k=img||tt;if(k&&seen.has(k))it.remove();else if(k)seen.add(k);});
|
| 311 |
-
if(w.querySelectorAll('.slider-item').length>=6)return;
|
| 312 |
-
w.remove();
|
| 313 |
-
}
|
| 314 |
-
let sh=await fetch('/api/shorts?refresh=1').then(r=>r.json()).catch(()=>[]);if(!sh.length)return;
|
| 315 |
-
let wrap=document.createElement('div');wrap.className='slider-wrap';wrap.id='shorts-final6-stable';let h='<div class="slider-header"><span class="slider-label">📱 Shorts Dân trí & SKĐS</span><span class="slider-note">Mới nhất</span></div><div class="slider-track">';
|
| 316 |
-
sh.slice(0,30).forEach((a,i)=>{h+=`<div class="slider-item shorts-item" onclick="openTikTok('shorts',${i})"><div class="slider-thumb shorts-thumb">${a.img?`<img src="${esc(a.img)}">`:''}<div class="card-play">▶</div></div><div class="slider-title">${esc(a.title)}</div></div>`});h+='</div>';wrap.innerHTML=h;
|
| 317 |
-
let comp=document.querySelector('.ai-compose')||document.getElementById('view-home').firstChild;if(comp)comp.after(wrap);else document.getElementById('view-home').prepend(wrap);
|
| 318 |
-
}
|
| 319 |
function renderLiveTopicWall(){let home=document.getElementById('view-home');if(!home||!liveTopicWall.length)return;document.getElementById('ai-wall-topic-live')?.remove();let wrap=document.createElement('div');wrap.id='ai-wall-topic-live';wrap.className='ai-wall-topic-live';let h='<div class="slider-header"><span class="slider-label">🧱 Tường AI mới</span><span class="slider-note">Tổng hợp từ web</span></div><div class="slider-track">';liveTopicWall.slice(0,20).forEach((p,i)=>{h+=`<div class="wall-item"><div class="wall-thumb">${p.img?`<img src="${esc(p.img)}">`:''}</div><div class="wall-title">${esc(p.title)}</div><div class="wall-text">${esc(p.text)}</div><div class="wall-actions"><button class="primary" onclick="readLiveTopicWall(${i})">Xem</button></div></div>`});h+='</div>';wrap.innerHTML=h;let comp=document.querySelector('.ai-compose');if(comp)comp.after(wrap);else home.prepend(wrap);}
|
| 320 |
window.readLiveTopicWall=function(i){let p=liveTopicWall[i];if(!p)return;showView('view-article');let imgs=(p.images||[]).filter(Boolean);let gal=imgs.length?'<div class="ai-wall-gallery">'+imgs.slice(0,12).map(u=>`<img src="${esc(u)}" loading="lazy">`).join('')+'</div>':(p.img?`<img class="article-img" src="${esc(p.img)}">`:'');document.getElementById('view-article').innerHTML=`<button class="back-btn" onclick="switchCat('home')">← Quay lại</button><div class="article-view"><span class="badge badge-ai">AI</span><h1 class="article-title">${esc(p.title)}</h1>${gal}<p class="article-p" style="white-space:pre-wrap">${esc(p.text)}</p><div class="article-actions"><button onclick="shareAI?shareAI(${JSON.stringify(p).replace(/"/g,'"')},false):navigator.clipboard.writeText(location.href)">📤 Chia sẻ</button></div></div>`;window.scrollTo(0,0)};
|
| 321 |
window.createTopicPostFinal5=async function(){let inp=document.getElementById('ai-topic-input-final5');let topic=(inp&&inp.value||'').trim();if(!topic)return alert('Nhập chủ đề trước');let btn=document.getElementById('ai-topic-btn-final5');if(btn){btn.disabled=true;btn.textContent='Đang crawl và tổng hợp...'}try{let r=await fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic})});let j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');liveTopicWall.unshift(j.post);if(inp)inp.value='';renderLiveTopicWall();readLiveTopicWall(0);alert('Đã tạo bài tổng hợp từ nội dung web và đăng lên Tường AI.');}catch(e){alert(e.message)}finally{if(btn){btn.disabled=false;btn.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen'}}};
|
| 322 |
-
setInterval(()=>{document.querySelectorAll('#ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4').forEach(e=>(e.closest('.topic-final3,.topic-final4,.ai-compose-row')||e).remove());let b=document.getElementById('ai-topic-btn-final5');if(b){b.style.display='block';b.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen';}ensureNewsShortsHome();},1200);
|
| 323 |
-
setTimeout(ensureNewsShortsHome,1200);
|
| 324 |
})();
|
| 325 |
</script>
|
| 326 |
'''
|
|
|
|
| 1 |
+
"""Final6: robust topic synthesis, stable shorts, hot topic hashtags."""
|
| 2 |
+
import re, time, json, os, threading, html as html_lib
|
| 3 |
from urllib.parse import quote, urlparse, parse_qs, unquote
|
| 4 |
import requests
|
| 5 |
from bs4 import BeautifulSoup
|
| 6 |
import ai_runtime_final5 as f5
|
| 7 |
from ai_runtime_final5 import app, rt, HTMLResponse, JSONResponse, Request, Query
|
| 8 |
|
| 9 |
+
_PATCH={('/api/topic_post','POST'),('/api/shorts','GET'),('/api/hot_topics','GET'),('/','GET')}
|
|
|
|
| 10 |
app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
|
| 11 |
|
| 12 |
_TOPIC_CACHE={}
|
| 13 |
+
_HOT_CACHE={"t":0,"d":[]}
|
| 14 |
_SHORTS_CACHE_FINAL6={"t":0,"d":[]}
|
| 15 |
_TRANSLATE_CACHE_PATH="/data/title_vi_cache.json" if os.path.isdir('/data') else "/app/data/title_vi_cache.json"
|
| 16 |
_translate_lock=threading.Lock()
|
| 17 |
YOUTUBE_HANDLES=["baodantri7941","baosuckhoedoisongboyte"]
|
| 18 |
UA={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Accept-Language":"vi,en;q=0.8"}
|
| 19 |
+
STOP_WORDS=set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật'.split())
|
| 20 |
|
| 21 |
+
def clean(s):return re.sub(r"\s+"," ",html_lib.unescape(str(s or ""))).strip()
|
|
|
|
|
|
|
| 22 |
def _domain(u):
|
| 23 |
try:return urlparse(u or '').netloc.replace('www.','')
|
| 24 |
except Exception:return ''
|
|
|
|
| 29 |
with open(_TRANSLATE_CACHE_PATH,'r',encoding='utf-8') as f:return json.load(f)
|
| 30 |
except Exception:pass
|
| 31 |
return {}
|
|
|
|
| 32 |
def _save_title_cache(db):
|
| 33 |
try:
|
| 34 |
+
os.makedirs(os.path.dirname(_TRANSLATE_CACHE_PATH),exist_ok=True);tmp=_TRANSLATE_CACHE_PATH+'.tmp'
|
|
|
|
| 35 |
with open(tmp,'w',encoding='utf-8') as f:json.dump(db,f,ensure_ascii=False)
|
| 36 |
os.replace(tmp,_TRANSLATE_CACHE_PATH)
|
| 37 |
except Exception:pass
|
|
|
|
| 39 |
def _looks_vietnamese(s):
|
| 40 |
s=s or ''
|
| 41 |
if re.search(r'[àáạảãâầấậẩẫăằắặẳẵèéẹẻẽêềếệểễìíịỉĩòóọỏõôồốộổỗơờớợởỡùúụủũưừứựửữỳýỵỷỹđ]',s,re.I):return True
|
|
|
|
| 42 |
low=' '+s.lower()+' '
|
| 43 |
+
return any(w in low for w in [' và ',' của ',' người ',' tại ',' trong ',' với ',' không ',' được ',' công an ',' bệnh viện ',' học sinh ',' tài xế ',' bóng đá ',' tin tức ',' sức khỏe '])
|
|
|
|
| 44 |
def _translate_title_vi(title):
|
| 45 |
title=clean(title)
|
| 46 |
+
if not title or _looks_vietnamese(title):return title
|
|
|
|
| 47 |
with _translate_lock:
|
| 48 |
db=_load_title_cache()
|
| 49 |
if title in db:return db[title]
|
|
|
|
| 51 |
try:
|
| 52 |
r=requests.get('https://translate.googleapis.com/translate_a/single',params={'client':'gtx','sl':'auto','tl':'vi','dt':'t','q':title},headers=UA,timeout=8)
|
| 53 |
if r.status_code==200:
|
| 54 |
+
data=r.json();vi=''.join(part[0] for part in data[0] if part and part[0]).strip() or title
|
|
|
|
| 55 |
except Exception:pass
|
| 56 |
vi=clean(vi)
|
| 57 |
with _translate_lock:
|
| 58 |
db=_load_title_cache();db[title]=vi;_save_title_cache(db)
|
| 59 |
return vi
|
| 60 |
|
| 61 |
+
# ===== Hot topics / hashtags =====
|
| 62 |
+
def _keywords_from_title(title):
|
| 63 |
+
title=clean(re.sub(r'\s+-\s+.*$','',title))
|
| 64 |
+
words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in STOP_WORDS]
|
| 65 |
+
phrases=[]
|
| 66 |
+
# Prefer named-ish 2-4 word chunks from title
|
| 67 |
+
for n in (4,3,2):
|
| 68 |
+
for i in range(0,max(0,len(words)-n+1)):
|
| 69 |
+
ph=' '.join(words[i:i+n]).strip()
|
| 70 |
+
if len(ph)>=8:phrases.append(ph)
|
| 71 |
+
if words:phrases.append(' '.join(words[:5]))
|
| 72 |
+
return phrases[:4]
|
| 73 |
+
|
| 74 |
+
def _hot_topics():
|
| 75 |
+
now=time.time()
|
| 76 |
+
if _HOT_CACHE['d'] and now-_HOT_CACHE['t']<900:return _HOT_CACHE['d']
|
| 77 |
+
topics=[];seen=set()
|
| 78 |
+
feeds=[
|
| 79 |
+
'https://news.google.com/rss?hl=vi&gl=VN&ceid=VN:vi',
|
| 80 |
+
'https://news.google.com/rss/headlines/section/topic/NATION?hl=vi&gl=VN&ceid=VN:vi',
|
| 81 |
+
'https://news.google.com/rss/headlines/section/topic/BUSINESS?hl=vi&gl=VN&ceid=VN:vi',
|
| 82 |
+
'https://news.google.com/rss/headlines/section/topic/SPORTS?hl=vi&gl=VN&ceid=VN:vi',
|
| 83 |
+
'https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=vi&gl=VN&ceid=VN:vi'
|
| 84 |
+
]
|
| 85 |
+
for feed in feeds:
|
| 86 |
+
try:
|
| 87 |
+
r=requests.get(feed,headers=UA,timeout=10);r.encoding='utf-8'
|
| 88 |
+
soup=BeautifulSoup(r.text,'xml')
|
| 89 |
+
for it in soup.find_all('item')[:15]:
|
| 90 |
+
title=clean(it.find('title').get_text(' ',strip=True) if it.find('title') else '')
|
| 91 |
+
for kw in _keywords_from_title(title):
|
| 92 |
+
key=kw.lower()
|
| 93 |
+
if key not in seen and len(kw)<=60:
|
| 94 |
+
seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
|
| 95 |
+
if len(topics)>=24:break
|
| 96 |
+
if len(topics)>=24:break
|
| 97 |
+
except Exception:pass
|
| 98 |
+
if len(topics)>=24:break
|
| 99 |
+
# Stable fallback if Google RSS fails
|
| 100 |
+
for kw in ['AI trong giáo dục','World Cup 2026','kinh tế Việt Nam','biến đổi khí hậu','giá vàng','bóng đá Việt Nam','an ninh mạng','xe điện','sức khỏe tinh thần','thị trường chứng khoán']:
|
| 101 |
+
if kw.lower() not in seen:topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
|
| 102 |
+
_HOT_CACHE.update({'t':now,'d':topics[:24]})
|
| 103 |
+
return _HOT_CACHE['d']
|
| 104 |
+
|
| 105 |
+
@app.get('/api/hot_topics')
|
| 106 |
+
def api_hot_topics():return JSONResponse({'topics':_hot_topics()})
|
| 107 |
+
|
| 108 |
+
# ===== Topic web research =====
|
| 109 |
def _unwrap_ddg_href(href):
|
| 110 |
if not href:return ''
|
| 111 |
if href.startswith('//duckduckgo.com/l/?') or 'duckduckgo.com/l/?' in href:
|
|
|
|
| 122 |
for res in soup.select('.result'):
|
| 123 |
a=res.select_one('.result__title a') or res.find('a',href=True)
|
| 124 |
if not a:continue
|
| 125 |
+
link=_unwrap_ddg_href(a.get('href',''));title=clean(a.get_text(' ',strip=True));snippet=clean((res.select_one('.result__snippet') or res).get_text(' ',strip=True))
|
|
|
|
|
|
|
| 126 |
if not link.startswith('http') or link in seen:continue
|
| 127 |
if any(bad in link for bad in ['duckduckgo.com','youtube.com','facebook.com','tiktok.com']):continue
|
| 128 |
seen.add(link);items.append({'title':title,'url':link,'source':_domain(link),'snippet':snippet})
|
|
|
|
| 150 |
try:
|
| 151 |
r=requests.get(url,headers=UA,timeout=16,allow_redirects=True)
|
| 152 |
if r.status_code>=400:return ''
|
| 153 |
+
r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
|
|
|
|
| 154 |
for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe','svg']):tag.decompose()
|
| 155 |
candidates=[]
|
| 156 |
for sel in ['article','main','.article-content','.detail-content','.singular-content','.fck_detail','.content-detail','.entry-content','.story-body','.knc-content']:
|
|
|
|
| 161 |
ps=[]
|
| 162 |
for el in best.find_all(['p','h2','h3'],recursive=True):
|
| 163 |
t=clean(el.get_text(' ',strip=True))
|
| 164 |
+
if len(t)>45 and not any(x in t.lower() for x in ['đăng ký nhận tin','theo dõi chúng tôi','chuyên mục','xem thêm','tin liên quan']):ps.append(t)
|
|
|
|
| 165 |
if sum(len(x) for x in ps)>max_chars:break
|
| 166 |
return '\n'.join(ps)[:max_chars]
|
| 167 |
except Exception:return ''
|
|
|
|
| 188 |
def _web_research_context(topic):
|
| 189 |
now=time.time();key=topic.lower().strip()
|
| 190 |
if key in _TOPIC_CACHE and now-_TOPIC_CACHE[key]['t']<900:return _TOPIC_CACHE[key]['d']
|
|
|
|
| 191 |
items=_ddg_search(topic,10)
|
| 192 |
if len(items)<4:
|
| 193 |
seen={i['url'] for i in items}
|
| 194 |
for it in _google_news_items(topic,8):
|
| 195 |
+
if it['url'] not in seen:items.append(it);seen.add(it['url'])
|
|
|
|
| 196 |
blocks=[];sources=[]
|
| 197 |
for it in items[:10]:
|
| 198 |
text=_scrape_article_text(it['url'],6500)
|
|
|
|
| 215 |
async def topic_post_synthesis(request:Request):
|
| 216 |
body=await request.json();topic=clean(body.get('topic',''))
|
| 217 |
if not topic:return JSONResponse({'error':'missing topic'},status_code=400)
|
| 218 |
+
img=_topic_image(topic);research=_web_research_context(topic);context=research.get('context','');sources=research.get('sources',[])
|
|
|
|
|
|
|
|
|
|
| 219 |
prompt=f"""Bạn là biên tập viên VNEWS. Người dùng chọn chủ đề: "{topic}".
|
| 220 |
|
| 221 |
Dưới đây là NỘI DUNG các bài viết/đoạn mô tả đã crawl từ internet. Hãy đọc hiểu và TỔNG HỢP thành MỘT BÀI VIẾT HOÀN CHỈNH. Tuyệt đối không bê nguyên văn, không xếp danh sách tiêu đề thành bài viết, không viết kiểu trả lời chat.
|
|
|
|
| 239 |
body=block.split('NỘI DUNG BÀI VIẾT ĐÃ CRAWL:')[-1].split('ĐOẠN MÔ TẢ TỪ KẾT QUẢ TÌM KIẾM:')[-1].strip()
|
| 240 |
if len(body)>120:parts.append(body)
|
| 241 |
joined='\n\n'.join(parts)[:7000]
|
| 242 |
+
text=(f"{topic}: những điểm chính cần biết\n\n{topic} đang thu hút sự chú ý vì liên quan đến nhiều khía cạnh thực tế. Tổng hợp từ các nội dung thu thập được, có thể nhìn vấn đề qua bối cảnh, tác động và những điểm cần theo dõi.\n\n"+(joined if joined else 'Hiện dữ liệu crawl còn hạn chế, bài viết này chỉ đưa ra bối cảnh tổng quan và các điểm cần theo dõi tiếp.')+"\n\nNguồn tham khảo: "+', '.join(sorted({s.get('via','') for s in sources if s.get('via')})))
|
| 243 |
+
post=f5.base.make_post(topic,text,img,'','topic_web_synthesis',sources=[s for s in sources if s.get('url')]);post['images']=[img]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
posts=f5.base._load_ai_wall();posts.insert(0,post);f5.base._save_ai_wall(posts)
|
| 245 |
return JSONResponse({'post':post})
|
| 246 |
|
| 247 |
+
# ===== Stable newest Shorts =====
|
| 248 |
def _yt_ytdlp(handle,count=30):
|
| 249 |
try:
|
| 250 |
import yt_dlp
|
| 251 |
urls=[f'https://www.youtube.com/@{handle}/shorts',f'https://www.youtube.com/@{handle}/videos']
|
| 252 |
+
out=[];seen=set();opts={'quiet':True,'extract_flat':True,'skip_download':True,'playlistend':count,'ignoreerrors':True,'no_warnings':True,'extractor_args':{'youtube':{'player_client':['web']}}}
|
|
|
|
| 253 |
for url in urls:
|
| 254 |
+
with yt_dlp.YoutubeDL(opts) as ydl:info=ydl.extract_info(url,download=False)
|
|
|
|
| 255 |
for e in (info or {}).get('entries') or []:
|
| 256 |
vid=e.get('id') or ''
|
| 257 |
if not re.match(r'^[A-Za-z0-9_-]{11}$',vid) or vid in seen:continue
|
| 258 |
title=e.get('title') or 'YouTube Short'
|
| 259 |
+
if url.endswith('/videos') and '#short' not in title.lower() and 'shorts' not in title.lower():continue
|
|
|
|
| 260 |
seen.add(vid);out.append({'title':title,'link':'https://www.youtube.com/watch?v='+vid,'img':'https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg','source':'yt','id':vid,'channel':handle})
|
| 261 |
if len(out)>=count:break
|
| 262 |
if len(out)>=count:break
|
| 263 |
return out
|
| 264 |
except Exception:return []
|
|
|
|
| 265 |
def _yt_html(handle,count=30):
|
| 266 |
out=[];seen=set()
|
| 267 |
for suffix in ['shorts','videos']:
|
|
|
|
| 270 |
for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
|
| 271 |
vid=m.group(1)
|
| 272 |
if vid in seen:continue
|
| 273 |
+
snip=html[max(0,m.start()-1200):m.start()+2200];title='YouTube Short'
|
|
|
|
| 274 |
mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip) or re.search(r'"accessibilityText":"([^"]+)"',snip)
|
| 275 |
if mt:title=clean(mt.group(1).replace('\\n',' '))
|
| 276 |
if suffix=='videos' and '#short' not in title.lower() and 'shorts' not in title.lower():continue
|
|
|
|
| 279 |
except Exception:pass
|
| 280 |
if len(out)>=count:break
|
| 281 |
return out[:count]
|
|
|
|
| 282 |
def _fallback_shorts():
|
| 283 |
try:return f5._fallback_shorts()
|
| 284 |
except Exception:return []
|
|
|
|
| 285 |
@app.get('/api/shorts')
|
| 286 |
def api_shorts_final6(refresh:int=Query(default=0)):
|
| 287 |
now=time.time()
|
| 288 |
+
if not refresh and _SHORTS_CACHE_FINAL6['d'] and now-_SHORTS_CACHE_FINAL6['t']<600:return JSONResponse(_SHORTS_CACHE_FINAL6['d'])
|
|
|
|
| 289 |
raw=[]
|
| 290 |
+
for h in YOUTUBE_HANDLES:raw.extend(_yt_ytdlp(h,30) or _yt_html(h,30))
|
|
|
|
|
|
|
| 291 |
raw.extend(_fallback_shorts())
|
| 292 |
seen=set();out=[]
|
| 293 |
for v in raw:
|
| 294 |
vid=v.get('id') or ''
|
| 295 |
if not vid:
|
| 296 |
+
m=re.search(r'(?:v=|shorts/|youtu\.be/)([A-Za-z0-9_-]{11})',v.get('link',''));vid=m.group(1) if m else ''
|
| 297 |
+
title=_translate_title_vi(v.get('title') or 'YouTube Short');key=vid or re.sub(r'\W+','',title.lower())[:80]
|
|
|
|
|
|
|
| 298 |
if not key or key in seen:continue
|
| 299 |
+
seen.add(key);item=dict(v);item['id']=vid;item['title']=title
|
| 300 |
+
if vid:item['link']='https://www.youtube.com/watch?v='+vid;item['img']='https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg'
|
| 301 |
+
item['source']='yt';out.append(item)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
if len(out)>=40:break
|
| 303 |
_SHORTS_CACHE_FINAL6.update({'t':now,'d':out})
|
| 304 |
return JSONResponse(out)
|
| 305 |
|
| 306 |
FINAL6_INJECT=r'''
|
| 307 |
<style>
|
| 308 |
+
#ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4{display:none!important}.topic-final5{display:flex!important}.ai-wall-topic-live{margin:6px 4px;background:#1a1a1a;border:1px solid #2a2a2a;border-radius:8px;overflow:hidden}.hot-topic-row{display:flex;gap:6px;overflow-x:auto;padding:4px 0}.hot-chip{flex:0 0 auto;background:#222;border:1px solid #333;color:#ddd;border-radius:16px;padding:5px 10px;font-size:11px;cursor:pointer}.hot-chip:active{transform:scale(.96)}
|
| 309 |
</style>
|
| 310 |
<script>
|
| 311 |
(function(){
|
| 312 |
function esc(s){return String(s||'').replace(/[&<>"']/g,m=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[m]));}
|
| 313 |
let liveTopicWall=[];
|
| 314 |
+
async function ensureHotTopics(){let inp=document.getElementById('ai-topic-input-final5');if(!inp||document.getElementById('hot-topic-row-final6'))return;let row=document.createElement('div');row.id='hot-topic-row-final6';row.className='hot-topic-row';row.innerHTML='<span style="color:#777;font-size:11px;padding:5px 0">Đang tải từ khóa nóng...</span>';inp.insertAdjacentElement('afterend',row);let j=await fetch('/api/hot_topics').then(r=>r.json()).catch(()=>({topics:[]}));let topics=j.topics||[];row.innerHTML=topics.slice(0,18).map(t=>`<button class="hot-chip" onclick="document.getElementById('ai-topic-input-final5').value='${esc(t.topic).replace(/'/g,'\\\'')}';document.getElementById('ai-topic-input-final5').focus();">${esc(t.label)}</button>`).join('')||'';}
|
| 315 |
+
async function ensureNewsShortsHome(){if(!document.getElementById('view-home')?.classList.contains('active'))return;let labels=[...document.querySelectorAll('.slider-wrap .slider-label')];let wraps=labels.filter(l=>/shorts|short /i.test(l.textContent||'')&&!/short ai/i.test(l.textContent||'')).map(l=>l.closest('.slider-wrap')).filter(Boolean);wraps.forEach((w,i)=>{if(i>0)w.remove();});let w=wraps[0];if(w){let seen=new Set();[...w.querySelectorAll('.slider-item')].forEach(it=>{let img=it.querySelector('img')?.src||'';let tt=(it.querySelector('.slider-title')?.textContent||'').trim().toLowerCase();let k=img||tt;if(k&&seen.has(k))it.remove();else if(k)seen.add(k);});if(w.querySelectorAll('.slider-item').length>=6)return;w.remove();}let sh=await fetch('/api/shorts?refresh=1').then(r=>r.json()).catch(()=>[]);if(!sh.length)return;let wrap=document.createElement('div');wrap.className='slider-wrap';wrap.id='shorts-final6-stable';let h='<div class="slider-header"><span class="slider-label">📱 Shorts Dân trí & SKĐS</span><span class="slider-note">Mới nhất</span></div><div class="slider-track">';sh.slice(0,30).forEach((a,i)=>{h+=`<div class="slider-item shorts-item" onclick="openTikTok('shorts',${i})"><div class="slider-thumb shorts-thumb">${a.img?`<img src="${esc(a.img)}">`:''}<div class="card-play">▶</div></div><div class="slider-title">${esc(a.title)}</div></div>`});h+='</div>';wrap.innerHTML=h;let comp=document.querySelector('.ai-compose')||document.getElementById('view-home').firstChild;if(comp)comp.after(wrap);else document.getElementById('view-home').prepend(wrap);}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
function renderLiveTopicWall(){let home=document.getElementById('view-home');if(!home||!liveTopicWall.length)return;document.getElementById('ai-wall-topic-live')?.remove();let wrap=document.createElement('div');wrap.id='ai-wall-topic-live';wrap.className='ai-wall-topic-live';let h='<div class="slider-header"><span class="slider-label">🧱 Tường AI mới</span><span class="slider-note">Tổng hợp từ web</span></div><div class="slider-track">';liveTopicWall.slice(0,20).forEach((p,i)=>{h+=`<div class="wall-item"><div class="wall-thumb">${p.img?`<img src="${esc(p.img)}">`:''}</div><div class="wall-title">${esc(p.title)}</div><div class="wall-text">${esc(p.text)}</div><div class="wall-actions"><button class="primary" onclick="readLiveTopicWall(${i})">Xem</button></div></div>`});h+='</div>';wrap.innerHTML=h;let comp=document.querySelector('.ai-compose');if(comp)comp.after(wrap);else home.prepend(wrap);}
|
| 317 |
window.readLiveTopicWall=function(i){let p=liveTopicWall[i];if(!p)return;showView('view-article');let imgs=(p.images||[]).filter(Boolean);let gal=imgs.length?'<div class="ai-wall-gallery">'+imgs.slice(0,12).map(u=>`<img src="${esc(u)}" loading="lazy">`).join('')+'</div>':(p.img?`<img class="article-img" src="${esc(p.img)}">`:'');document.getElementById('view-article').innerHTML=`<button class="back-btn" onclick="switchCat('home')">← Quay lại</button><div class="article-view"><span class="badge badge-ai">AI</span><h1 class="article-title">${esc(p.title)}</h1>${gal}<p class="article-p" style="white-space:pre-wrap">${esc(p.text)}</p><div class="article-actions"><button onclick="shareAI?shareAI(${JSON.stringify(p).replace(/"/g,'"')},false):navigator.clipboard.writeText(location.href)">📤 Chia sẻ</button></div></div>`;window.scrollTo(0,0)};
|
| 318 |
window.createTopicPostFinal5=async function(){let inp=document.getElementById('ai-topic-input-final5');let topic=(inp&&inp.value||'').trim();if(!topic)return alert('Nhập chủ đề trước');let btn=document.getElementById('ai-topic-btn-final5');if(btn){btn.disabled=true;btn.textContent='Đang crawl và tổng hợp...'}try{let r=await fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic})});let j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');liveTopicWall.unshift(j.post);if(inp)inp.value='';renderLiveTopicWall();readLiveTopicWall(0);alert('Đã tạo bài tổng hợp từ nội dung web và đăng lên Tường AI.');}catch(e){alert(e.message)}finally{if(btn){btn.disabled=false;btn.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen'}}};
|
| 319 |
+
setInterval(()=>{document.querySelectorAll('#ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4').forEach(e=>(e.closest('.topic-final3,.topic-final4,.ai-compose-row')||e).remove());let b=document.getElementById('ai-topic-btn-final5');if(b){b.style.display='block';b.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen';}ensureHotTopics();ensureNewsShortsHome();},1200);setTimeout(()=>{ensureHotTopics();ensureNewsShortsHome();},1200);
|
|
|
|
| 320 |
})();
|
| 321 |
</script>
|
| 322 |
'''
|