bep40 commited on
Commit
8bec409
·
verified ·
1 Parent(s): 57f39a2

Add hot Google keywords under topic box + improve topic research guidance

Browse files
Files changed (1) hide show
  1. ai_runtime_final6.py +81 -85
ai_runtime_final6.py CHANGED
@@ -1,25 +1,24 @@
1
- """Final6: robust topic synthesis from real web article bodies + stable latest Dantri/SKDS shorts."""
2
- import re, time, json, os, threading, html as html_lib, subprocess
3
  from urllib.parse import quote, urlparse, parse_qs, unquote
4
  import requests
5
  from bs4 import BeautifulSoup
6
  import ai_runtime_final5 as f5
7
  from ai_runtime_final5 import app, rt, HTMLResponse, JSONResponse, Request, Query
8
 
9
- # Override endpoints/root after previous runtime patches.
10
- _PATCH={('/api/topic_post','POST'),('/api/shorts','GET'),('/','GET')}
11
  app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
12
 
13
  _TOPIC_CACHE={}
 
14
  _SHORTS_CACHE_FINAL6={"t":0,"d":[]}
15
  _TRANSLATE_CACHE_PATH="/data/title_vi_cache.json" if os.path.isdir('/data') else "/app/data/title_vi_cache.json"
16
  _translate_lock=threading.Lock()
17
  YOUTUBE_HANDLES=["baodantri7941","baosuckhoedoisongboyte"]
18
  UA={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Accept-Language":"vi,en;q=0.8"}
 
19
 
20
- def clean(s):
21
- return re.sub(r"\s+"," ",html_lib.unescape(str(s or ""))).strip()
22
-
23
  def _domain(u):
24
  try:return urlparse(u or '').netloc.replace('www.','')
25
  except Exception:return ''
@@ -30,11 +29,9 @@ def _load_title_cache():
30
  with open(_TRANSLATE_CACHE_PATH,'r',encoding='utf-8') as f:return json.load(f)
31
  except Exception:pass
32
  return {}
33
-
34
  def _save_title_cache(db):
35
  try:
36
- os.makedirs(os.path.dirname(_TRANSLATE_CACHE_PATH),exist_ok=True)
37
- tmp=_TRANSLATE_CACHE_PATH+'.tmp'
38
  with open(tmp,'w',encoding='utf-8') as f:json.dump(db,f,ensure_ascii=False)
39
  os.replace(tmp,_TRANSLATE_CACHE_PATH)
40
  except Exception:pass
@@ -42,14 +39,11 @@ def _save_title_cache(db):
42
  def _looks_vietnamese(s):
43
  s=s or ''
44
  if re.search(r'[àáạảãâầấậẩẫăằắặẳẵèéẹẻẽêềếệểễìíịỉĩòóọỏõôồốộổỗơờớợởỡùúụủũưừứựửữỳýỵỷỹđ]',s,re.I):return True
45
- vi_words=[' và ',' của ',' người ',' tại ',' trong ',' với ',' không ',' được ',' sau ',' trước ',' công an ',' bệnh viện ',' học sinh ',' tài xế ',' bóng đá ',' tin tức ',' sức khỏe ']
46
  low=' '+s.lower()+' '
47
- return any(w in low for w in vi_words)
48
-
49
  def _translate_title_vi(title):
50
  title=clean(title)
51
- if not title:return title
52
- if _looks_vietnamese(title):return title
53
  with _translate_lock:
54
  db=_load_title_cache()
55
  if title in db:return db[title]
@@ -57,15 +51,61 @@ def _translate_title_vi(title):
57
  try:
58
  r=requests.get('https://translate.googleapis.com/translate_a/single',params={'client':'gtx','sl':'auto','tl':'vi','dt':'t','q':title},headers=UA,timeout=8)
59
  if r.status_code==200:
60
- data=r.json()
61
- vi=''.join(part[0] for part in data[0] if part and part[0]).strip() or title
62
  except Exception:pass
63
  vi=clean(vi)
64
  with _translate_lock:
65
  db=_load_title_cache();db[title]=vi;_save_title_cache(db)
66
  return vi
67
 
68
- # ===== Topic web research: use direct search result URLs, not only RSS titles =====
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
  def _unwrap_ddg_href(href):
70
  if not href:return ''
71
  if href.startswith('//duckduckgo.com/l/?') or 'duckduckgo.com/l/?' in href:
@@ -82,9 +122,7 @@ def _ddg_search(topic, limit=10):
82
  for res in soup.select('.result'):
83
  a=res.select_one('.result__title a') or res.find('a',href=True)
84
  if not a:continue
85
- link=_unwrap_ddg_href(a.get('href',''))
86
- title=clean(a.get_text(' ',strip=True))
87
- snippet=clean((res.select_one('.result__snippet') or res).get_text(' ',strip=True))
88
  if not link.startswith('http') or link in seen:continue
89
  if any(bad in link for bad in ['duckduckgo.com','youtube.com','facebook.com','tiktok.com']):continue
90
  seen.add(link);items.append({'title':title,'url':link,'source':_domain(link),'snippet':snippet})
@@ -112,8 +150,7 @@ def _extract_article_text_bs(url, max_chars=6500):
112
  try:
113
  r=requests.get(url,headers=UA,timeout=16,allow_redirects=True)
114
  if r.status_code>=400:return ''
115
- r.encoding='utf-8'
116
- soup=BeautifulSoup(r.text,'lxml')
117
  for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe','svg']):tag.decompose()
118
  candidates=[]
119
  for sel in ['article','main','.article-content','.detail-content','.singular-content','.fck_detail','.content-detail','.entry-content','.story-body','.knc-content']:
@@ -124,8 +161,7 @@ def _extract_article_text_bs(url, max_chars=6500):
124
  ps=[]
125
  for el in best.find_all(['p','h2','h3'],recursive=True):
126
  t=clean(el.get_text(' ',strip=True))
127
- if len(t)>45 and not any(x in t.lower() for x in ['đăng ký nhận tin','theo dõi chúng tôi','chuyên mục','xem thêm','tin liên quan']):
128
- ps.append(t)
129
  if sum(len(x) for x in ps)>max_chars:break
130
  return '\n'.join(ps)[:max_chars]
131
  except Exception:return ''
@@ -152,13 +188,11 @@ def _scrape_article_text(url, max_chars=6500):
152
  def _web_research_context(topic):
153
  now=time.time();key=topic.lower().strip()
154
  if key in _TOPIC_CACHE and now-_TOPIC_CACHE[key]['t']<900:return _TOPIC_CACHE[key]['d']
155
- # Direct search first; RSS only supplements snippets/titles.
156
  items=_ddg_search(topic,10)
157
  if len(items)<4:
158
  seen={i['url'] for i in items}
159
  for it in _google_news_items(topic,8):
160
- if it['url'] not in seen:
161
- items.append(it);seen.add(it['url'])
162
  blocks=[];sources=[]
163
  for it in items[:10]:
164
  text=_scrape_article_text(it['url'],6500)
@@ -181,10 +215,7 @@ def _topic_image(topic):
181
  async def topic_post_synthesis(request:Request):
182
  body=await request.json();topic=clean(body.get('topic',''))
183
  if not topic:return JSONResponse({'error':'missing topic'},status_code=400)
184
- img=_topic_image(topic)
185
- research=_web_research_context(topic)
186
- context=research.get('context','')
187
- sources=research.get('sources',[])
188
  prompt=f"""Bạn là biên tập viên VNEWS. Người dùng chọn chủ đề: "{topic}".
189
 
190
  Dưới đây là NỘI DUNG các bài viết/đoạn mô tả đã crawl từ internet. Hãy đọc hiểu và TỔNG HỢP thành MỘT BÀI VIẾT HOÀN CHỈNH. Tuyệt đối không bê nguyên văn, không xếp danh sách tiêu đề thành bài viết, không viết kiểu trả lời chat.
@@ -208,37 +239,29 @@ Yêu cầu bắt buộc:
208
  body=block.split('NỘI DUNG BÀI VIẾT ĐÃ CRAWL:')[-1].split('ĐOẠN MÔ TẢ TỪ KẾT QUẢ TÌM KIẾM:')[-1].strip()
209
  if len(body)>120:parts.append(body)
210
  joined='\n\n'.join(parts)[:7000]
211
- text=(f"{topic}: những điểm chính cần biết\n\n"
212
- f"{topic} đang thu hút sự chú ý vì liên quan đến nhiều khía cạnh thực tế. Tổng hợp từ các nội dung thu thập được, có thể nhìn vấn đề qua bối cảnh, tác động những điểm cần theo dõi.\n\n"
213
- + (joined if joined else 'Hiện dữ liệu crawl còn hạn chế, bài viết này chỉ đưa ra bối cảnh tổng quan và các điểm cần theo dõi tiếp.') +
214
- "\n\nNguồn tham khảo: " + ', '.join(sorted({s.get('via','') for s in sources if s.get('via')})) )
215
- post=f5.base.make_post(topic,text,img,'','topic_web_synthesis',sources=[s for s in sources if s.get('url')])
216
- post['images']=[img]
217
  posts=f5.base._load_ai_wall();posts.insert(0,post);f5.base._save_ai_wall(posts)
218
  return JSONResponse({'post':post})
219
 
220
- # ===== Stable newest Dantri/SKDS Shorts =====
221
  def _yt_ytdlp(handle,count=30):
222
  try:
223
  import yt_dlp
224
  urls=[f'https://www.youtube.com/@{handle}/shorts',f'https://www.youtube.com/@{handle}/videos']
225
- out=[];seen=set()
226
- opts={'quiet':True,'extract_flat':True,'skip_download':True,'playlistend':count,'ignoreerrors':True,'no_warnings':True,'extractor_args':{'youtube':{'player_client':['web']}}}
227
  for url in urls:
228
- with yt_dlp.YoutubeDL(opts) as ydl:
229
- info=ydl.extract_info(url,download=False)
230
  for e in (info or {}).get('entries') or []:
231
  vid=e.get('id') or ''
232
  if not re.match(r'^[A-Za-z0-9_-]{11}$',vid) or vid in seen:continue
233
  title=e.get('title') or 'YouTube Short'
234
- if url.endswith('/videos') and '#short' not in title.lower() and 'shorts' not in title.lower():
235
- continue
236
  seen.add(vid);out.append({'title':title,'link':'https://www.youtube.com/watch?v='+vid,'img':'https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg','source':'yt','id':vid,'channel':handle})
237
  if len(out)>=count:break
238
  if len(out)>=count:break
239
  return out
240
  except Exception:return []
241
-
242
  def _yt_html(handle,count=30):
243
  out=[];seen=set()
244
  for suffix in ['shorts','videos']:
@@ -247,8 +270,7 @@ def _yt_html(handle,count=30):
247
  for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
248
  vid=m.group(1)
249
  if vid in seen:continue
250
- snip=html[max(0,m.start()-1200):m.start()+2200]
251
- title='YouTube Short'
252
  mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip) or re.search(r'"accessibilityText":"([^"]+)"',snip)
253
  if mt:title=clean(mt.group(1).replace('\\n',' '))
254
  if suffix=='videos' and '#short' not in title.lower() and 'shorts' not in title.lower():continue
@@ -257,70 +279,44 @@ def _yt_html(handle,count=30):
257
  except Exception:pass
258
  if len(out)>=count:break
259
  return out[:count]
260
-
261
  def _fallback_shorts():
262
  try:return f5._fallback_shorts()
263
  except Exception:return []
264
-
265
  @app.get('/api/shorts')
266
  def api_shorts_final6(refresh:int=Query(default=0)):
267
  now=time.time()
268
- if not refresh and _SHORTS_CACHE_FINAL6['d'] and now-_SHORTS_CACHE_FINAL6['t']<600:
269
- return JSONResponse(_SHORTS_CACHE_FINAL6['d'])
270
  raw=[]
271
- for h in YOUTUBE_HANDLES:
272
- got=_yt_ytdlp(h,30) or _yt_html(h,30)
273
- raw.extend(got)
274
  raw.extend(_fallback_shorts())
275
  seen=set();out=[]
276
  for v in raw:
277
  vid=v.get('id') or ''
278
  if not vid:
279
- m=re.search(r'(?:v=|shorts/|youtu\.be/)([A-Za-z0-9_-]{11})',v.get('link',''))
280
- vid=m.group(1) if m else ''
281
- title=_translate_title_vi(v.get('title') or 'YouTube Short')
282
- key=vid or re.sub(r'\W+','',title.lower())[:80]
283
  if not key or key in seen:continue
284
- seen.add(key)
285
- item=dict(v);item['id']=vid;item['title']=title
286
- if vid:
287
- item['link']='https://www.youtube.com/watch?v='+vid
288
- item['img']='https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg'
289
- item['source']='yt'
290
- out.append(item)
291
  if len(out)>=40:break
292
  _SHORTS_CACHE_FINAL6.update({'t':now,'d':out})
293
  return JSONResponse(out)
294
 
295
  FINAL6_INJECT=r'''
296
  <style>
297
- #ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4{display:none!important}.topic-final5{display:flex!important}.ai-wall-topic-live{margin:6px 4px;background:#1a1a1a;border:1px solid #2a2a2a;border-radius:8px;overflow:hidden}
298
  </style>
299
  <script>
300
  (function(){
301
  function esc(s){return String(s||'').replace(/[&<>"']/g,m=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[m]));}
302
  let liveTopicWall=[];
303
- async function ensureNewsShortsHome(){
304
- if(!document.getElementById('view-home')?.classList.contains('active'))return;
305
- let labels=[...document.querySelectorAll('.slider-wrap .slider-label')];
306
- let wraps=labels.filter(l=>/shorts|short /i.test(l.textContent||'')&&!/short ai/i.test(l.textContent||'')).map(l=>l.closest('.slider-wrap')).filter(Boolean);
307
- wraps.forEach((w,i)=>{if(i>0)w.remove();});
308
- let w=wraps[0];
309
- if(w){
310
- let seen=new Set();[...w.querySelectorAll('.slider-item')].forEach(it=>{let img=it.querySelector('img')?.src||'';let tt=(it.querySelector('.slider-title')?.textContent||'').trim().toLowerCase();let k=img||tt;if(k&&seen.has(k))it.remove();else if(k)seen.add(k);});
311
- if(w.querySelectorAll('.slider-item').length>=6)return;
312
- w.remove();
313
- }
314
- let sh=await fetch('/api/shorts?refresh=1').then(r=>r.json()).catch(()=>[]);if(!sh.length)return;
315
- let wrap=document.createElement('div');wrap.className='slider-wrap';wrap.id='shorts-final6-stable';let h='<div class="slider-header"><span class="slider-label">📱 Shorts Dân trí & SKĐS</span><span class="slider-note">Mới nhất</span></div><div class="slider-track">';
316
- sh.slice(0,30).forEach((a,i)=>{h+=`<div class="slider-item shorts-item" onclick="openTikTok('shorts',${i})"><div class="slider-thumb shorts-thumb">${a.img?`<img src="${esc(a.img)}">`:''}<div class="card-play">▶</div></div><div class="slider-title">${esc(a.title)}</div></div>`});h+='</div>';wrap.innerHTML=h;
317
- let comp=document.querySelector('.ai-compose')||document.getElementById('view-home').firstChild;if(comp)comp.after(wrap);else document.getElementById('view-home').prepend(wrap);
318
- }
319
  function renderLiveTopicWall(){let home=document.getElementById('view-home');if(!home||!liveTopicWall.length)return;document.getElementById('ai-wall-topic-live')?.remove();let wrap=document.createElement('div');wrap.id='ai-wall-topic-live';wrap.className='ai-wall-topic-live';let h='<div class="slider-header"><span class="slider-label">🧱 Tường AI mới</span><span class="slider-note">Tổng hợp từ web</span></div><div class="slider-track">';liveTopicWall.slice(0,20).forEach((p,i)=>{h+=`<div class="wall-item"><div class="wall-thumb">${p.img?`<img src="${esc(p.img)}">`:''}</div><div class="wall-title">${esc(p.title)}</div><div class="wall-text">${esc(p.text)}</div><div class="wall-actions"><button class="primary" onclick="readLiveTopicWall(${i})">Xem</button></div></div>`});h+='</div>';wrap.innerHTML=h;let comp=document.querySelector('.ai-compose');if(comp)comp.after(wrap);else home.prepend(wrap);}
320
  window.readLiveTopicWall=function(i){let p=liveTopicWall[i];if(!p)return;showView('view-article');let imgs=(p.images||[]).filter(Boolean);let gal=imgs.length?'<div class="ai-wall-gallery">'+imgs.slice(0,12).map(u=>`<img src="${esc(u)}" loading="lazy">`).join('')+'</div>':(p.img?`<img class="article-img" src="${esc(p.img)}">`:'');document.getElementById('view-article').innerHTML=`<button class="back-btn" onclick="switchCat('home')">← Quay lại</button><div class="article-view"><span class="badge badge-ai">AI</span><h1 class="article-title">${esc(p.title)}</h1>${gal}<p class="article-p" style="white-space:pre-wrap">${esc(p.text)}</p><div class="article-actions"><button onclick="shareAI?shareAI(${JSON.stringify(p).replace(/"/g,'&quot;')},false):navigator.clipboard.writeText(location.href)">📤 Chia sẻ</button></div></div>`;window.scrollTo(0,0)};
321
  window.createTopicPostFinal5=async function(){let inp=document.getElementById('ai-topic-input-final5');let topic=(inp&&inp.value||'').trim();if(!topic)return alert('Nhập chủ đề trước');let btn=document.getElementById('ai-topic-btn-final5');if(btn){btn.disabled=true;btn.textContent='Đang crawl và tổng hợp...'}try{let r=await fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic})});let j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');liveTopicWall.unshift(j.post);if(inp)inp.value='';renderLiveTopicWall();readLiveTopicWall(0);alert('Đã tạo bài tổng hợp từ nội dung web và đăng lên Tường AI.');}catch(e){alert(e.message)}finally{if(btn){btn.disabled=false;btn.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen'}}};
322
- setInterval(()=>{document.querySelectorAll('#ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4').forEach(e=>(e.closest('.topic-final3,.topic-final4,.ai-compose-row')||e).remove());let b=document.getElementById('ai-topic-btn-final5');if(b){b.style.display='block';b.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen';}ensureNewsShortsHome();},1200);
323
- setTimeout(ensureNewsShortsHome,1200);
324
  })();
325
  </script>
326
  '''
 
1
+ """Final6: robust topic synthesis, stable shorts, hot topic hashtags."""
2
+ import re, time, json, os, threading, html as html_lib
3
  from urllib.parse import quote, urlparse, parse_qs, unquote
4
  import requests
5
  from bs4 import BeautifulSoup
6
  import ai_runtime_final5 as f5
7
  from ai_runtime_final5 import app, rt, HTMLResponse, JSONResponse, Request, Query
8
 
9
+ _PATCH={('/api/topic_post','POST'),('/api/shorts','GET'),('/api/hot_topics','GET'),('/','GET')}
 
10
  app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
11
 
12
  _TOPIC_CACHE={}
13
+ _HOT_CACHE={"t":0,"d":[]}
14
  _SHORTS_CACHE_FINAL6={"t":0,"d":[]}
15
  _TRANSLATE_CACHE_PATH="/data/title_vi_cache.json" if os.path.isdir('/data') else "/app/data/title_vi_cache.json"
16
  _translate_lock=threading.Lock()
17
  YOUTUBE_HANDLES=["baodantri7941","baosuckhoedoisongboyte"]
18
  UA={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Accept-Language":"vi,en;q=0.8"}
19
+ STOP_WORDS=set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật'.split())
20
 
21
+ def clean(s):return re.sub(r"\s+"," ",html_lib.unescape(str(s or ""))).strip()
 
 
22
  def _domain(u):
23
  try:return urlparse(u or '').netloc.replace('www.','')
24
  except Exception:return ''
 
29
  with open(_TRANSLATE_CACHE_PATH,'r',encoding='utf-8') as f:return json.load(f)
30
  except Exception:pass
31
  return {}
 
32
  def _save_title_cache(db):
33
  try:
34
+ os.makedirs(os.path.dirname(_TRANSLATE_CACHE_PATH),exist_ok=True);tmp=_TRANSLATE_CACHE_PATH+'.tmp'
 
35
  with open(tmp,'w',encoding='utf-8') as f:json.dump(db,f,ensure_ascii=False)
36
  os.replace(tmp,_TRANSLATE_CACHE_PATH)
37
  except Exception:pass
 
39
  def _looks_vietnamese(s):
40
  s=s or ''
41
  if re.search(r'[àáạảãâầấậẩẫăằắặẳẵèéẹẻẽêềếệểễìíịỉĩòóọỏõôồốộổỗơờớợởỡùúụủũưừứựửữỳýỵỷỹđ]',s,re.I):return True
 
42
  low=' '+s.lower()+' '
43
+ return any(w in low for w in [' và ',' của ',' người ',' tại ',' trong ',' với ',' không ',' được ',' công an ',' bệnh viện ',' học sinh ',' tài xế ',' bóng đá ',' tin tức ',' sức khỏe '])
 
44
  def _translate_title_vi(title):
45
  title=clean(title)
46
+ if not title or _looks_vietnamese(title):return title
 
47
  with _translate_lock:
48
  db=_load_title_cache()
49
  if title in db:return db[title]
 
51
  try:
52
  r=requests.get('https://translate.googleapis.com/translate_a/single',params={'client':'gtx','sl':'auto','tl':'vi','dt':'t','q':title},headers=UA,timeout=8)
53
  if r.status_code==200:
54
+ data=r.json();vi=''.join(part[0] for part in data[0] if part and part[0]).strip() or title
 
55
  except Exception:pass
56
  vi=clean(vi)
57
  with _translate_lock:
58
  db=_load_title_cache();db[title]=vi;_save_title_cache(db)
59
  return vi
60
 
61
+ # ===== Hot topics / hashtags =====
62
+ def _keywords_from_title(title):
63
+ title=clean(re.sub(r'\s+-\s+.*$','',title))
64
+ words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in STOP_WORDS]
65
+ phrases=[]
66
+ # Prefer named-ish 2-4 word chunks from title
67
+ for n in (4,3,2):
68
+ for i in range(0,max(0,len(words)-n+1)):
69
+ ph=' '.join(words[i:i+n]).strip()
70
+ if len(ph)>=8:phrases.append(ph)
71
+ if words:phrases.append(' '.join(words[:5]))
72
+ return phrases[:4]
73
+
74
+ def _hot_topics():
75
+ now=time.time()
76
+ if _HOT_CACHE['d'] and now-_HOT_CACHE['t']<900:return _HOT_CACHE['d']
77
+ topics=[];seen=set()
78
+ feeds=[
79
+ 'https://news.google.com/rss?hl=vi&gl=VN&ceid=VN:vi',
80
+ 'https://news.google.com/rss/headlines/section/topic/NATION?hl=vi&gl=VN&ceid=VN:vi',
81
+ 'https://news.google.com/rss/headlines/section/topic/BUSINESS?hl=vi&gl=VN&ceid=VN:vi',
82
+ 'https://news.google.com/rss/headlines/section/topic/SPORTS?hl=vi&gl=VN&ceid=VN:vi',
83
+ 'https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=vi&gl=VN&ceid=VN:vi'
84
+ ]
85
+ for feed in feeds:
86
+ try:
87
+ r=requests.get(feed,headers=UA,timeout=10);r.encoding='utf-8'
88
+ soup=BeautifulSoup(r.text,'xml')
89
+ for it in soup.find_all('item')[:15]:
90
+ title=clean(it.find('title').get_text(' ',strip=True) if it.find('title') else '')
91
+ for kw in _keywords_from_title(title):
92
+ key=kw.lower()
93
+ if key not in seen and len(kw)<=60:
94
+ seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
95
+ if len(topics)>=24:break
96
+ if len(topics)>=24:break
97
+ except Exception:pass
98
+ if len(topics)>=24:break
99
+ # Stable fallback if Google RSS fails
100
+ for kw in ['AI trong giáo dục','World Cup 2026','kinh tế Việt Nam','biến đổi khí hậu','giá vàng','bóng đá Việt Nam','an ninh mạng','xe điện','sức khỏe tinh thần','thị trường chứng khoán']:
101
+ if kw.lower() not in seen:topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
102
+ _HOT_CACHE.update({'t':now,'d':topics[:24]})
103
+ return _HOT_CACHE['d']
104
+
105
+ @app.get('/api/hot_topics')
106
+ def api_hot_topics():return JSONResponse({'topics':_hot_topics()})
107
+
108
+ # ===== Topic web research =====
109
  def _unwrap_ddg_href(href):
110
  if not href:return ''
111
  if href.startswith('//duckduckgo.com/l/?') or 'duckduckgo.com/l/?' in href:
 
122
  for res in soup.select('.result'):
123
  a=res.select_one('.result__title a') or res.find('a',href=True)
124
  if not a:continue
125
+ link=_unwrap_ddg_href(a.get('href',''));title=clean(a.get_text(' ',strip=True));snippet=clean((res.select_one('.result__snippet') or res).get_text(' ',strip=True))
 
 
126
  if not link.startswith('http') or link in seen:continue
127
  if any(bad in link for bad in ['duckduckgo.com','youtube.com','facebook.com','tiktok.com']):continue
128
  seen.add(link);items.append({'title':title,'url':link,'source':_domain(link),'snippet':snippet})
 
150
  try:
151
  r=requests.get(url,headers=UA,timeout=16,allow_redirects=True)
152
  if r.status_code>=400:return ''
153
+ r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
 
154
  for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe','svg']):tag.decompose()
155
  candidates=[]
156
  for sel in ['article','main','.article-content','.detail-content','.singular-content','.fck_detail','.content-detail','.entry-content','.story-body','.knc-content']:
 
161
  ps=[]
162
  for el in best.find_all(['p','h2','h3'],recursive=True):
163
  t=clean(el.get_text(' ',strip=True))
164
+ if len(t)>45 and not any(x in t.lower() for x in ['đăng ký nhận tin','theo dõi chúng tôi','chuyên mục','xem thêm','tin liên quan']):ps.append(t)
 
165
  if sum(len(x) for x in ps)>max_chars:break
166
  return '\n'.join(ps)[:max_chars]
167
  except Exception:return ''
 
188
  def _web_research_context(topic):
189
  now=time.time();key=topic.lower().strip()
190
  if key in _TOPIC_CACHE and now-_TOPIC_CACHE[key]['t']<900:return _TOPIC_CACHE[key]['d']
 
191
  items=_ddg_search(topic,10)
192
  if len(items)<4:
193
  seen={i['url'] for i in items}
194
  for it in _google_news_items(topic,8):
195
+ if it['url'] not in seen:items.append(it);seen.add(it['url'])
 
196
  blocks=[];sources=[]
197
  for it in items[:10]:
198
  text=_scrape_article_text(it['url'],6500)
 
215
  async def topic_post_synthesis(request:Request):
216
  body=await request.json();topic=clean(body.get('topic',''))
217
  if not topic:return JSONResponse({'error':'missing topic'},status_code=400)
218
+ img=_topic_image(topic);research=_web_research_context(topic);context=research.get('context','');sources=research.get('sources',[])
 
 
 
219
  prompt=f"""Bạn là biên tập viên VNEWS. Người dùng chọn chủ đề: "{topic}".
220
 
221
  Dưới đây là NỘI DUNG các bài viết/đoạn mô tả đã crawl từ internet. Hãy đọc hiểu và TỔNG HỢP thành MỘT BÀI VIẾT HOÀN CHỈNH. Tuyệt đối không bê nguyên văn, không xếp danh sách tiêu đề thành bài viết, không viết kiểu trả lời chat.
 
239
  body=block.split('NỘI DUNG BÀI VIẾT ĐÃ CRAWL:')[-1].split('ĐOẠN MÔ TẢ TỪ KẾT QUẢ TÌM KIẾM:')[-1].strip()
240
  if len(body)>120:parts.append(body)
241
  joined='\n\n'.join(parts)[:7000]
242
+ text=(f"{topic}: những điểm chính cần biết\n\n{topic} đang thu hút sự chú ý vì liên quan đến nhiều khía cạnh thực tế. Tổng hợp từ các nội dung thu thập được, có thể nhìn vấn đề qua bối cảnh, tác động và những điểm cần theo dõi.\n\n"+(joined if joined else 'Hiện dữ liệu crawl còn hạn chế, bài viết này chỉ đưa ra bối cảnh tổng quan và các điểm cần theo dõi tiếp.')+"\n\nNguồn tham khảo: "+', '.join(sorted({s.get('via','') for s in sources if s.get('via')})))
243
+ post=f5.base.make_post(topic,text,img,'','topic_web_synthesis',sources=[s for s in sources if s.get('url')]);post['images']=[img]
 
 
 
 
244
  posts=f5.base._load_ai_wall();posts.insert(0,post);f5.base._save_ai_wall(posts)
245
  return JSONResponse({'post':post})
246
 
247
+ # ===== Stable newest Shorts =====
248
  def _yt_ytdlp(handle,count=30):
249
  try:
250
  import yt_dlp
251
  urls=[f'https://www.youtube.com/@{handle}/shorts',f'https://www.youtube.com/@{handle}/videos']
252
+ out=[];seen=set();opts={'quiet':True,'extract_flat':True,'skip_download':True,'playlistend':count,'ignoreerrors':True,'no_warnings':True,'extractor_args':{'youtube':{'player_client':['web']}}}
 
253
  for url in urls:
254
+ with yt_dlp.YoutubeDL(opts) as ydl:info=ydl.extract_info(url,download=False)
 
255
  for e in (info or {}).get('entries') or []:
256
  vid=e.get('id') or ''
257
  if not re.match(r'^[A-Za-z0-9_-]{11}$',vid) or vid in seen:continue
258
  title=e.get('title') or 'YouTube Short'
259
+ if url.endswith('/videos') and '#short' not in title.lower() and 'shorts' not in title.lower():continue
 
260
  seen.add(vid);out.append({'title':title,'link':'https://www.youtube.com/watch?v='+vid,'img':'https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg','source':'yt','id':vid,'channel':handle})
261
  if len(out)>=count:break
262
  if len(out)>=count:break
263
  return out
264
  except Exception:return []
 
265
  def _yt_html(handle,count=30):
266
  out=[];seen=set()
267
  for suffix in ['shorts','videos']:
 
270
  for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
271
  vid=m.group(1)
272
  if vid in seen:continue
273
+ snip=html[max(0,m.start()-1200):m.start()+2200];title='YouTube Short'
 
274
  mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip) or re.search(r'"accessibilityText":"([^"]+)"',snip)
275
  if mt:title=clean(mt.group(1).replace('\\n',' '))
276
  if suffix=='videos' and '#short' not in title.lower() and 'shorts' not in title.lower():continue
 
279
  except Exception:pass
280
  if len(out)>=count:break
281
  return out[:count]
 
282
  def _fallback_shorts():
283
  try:return f5._fallback_shorts()
284
  except Exception:return []
 
285
  @app.get('/api/shorts')
286
  def api_shorts_final6(refresh:int=Query(default=0)):
287
  now=time.time()
288
+ if not refresh and _SHORTS_CACHE_FINAL6['d'] and now-_SHORTS_CACHE_FINAL6['t']<600:return JSONResponse(_SHORTS_CACHE_FINAL6['d'])
 
289
  raw=[]
290
+ for h in YOUTUBE_HANDLES:raw.extend(_yt_ytdlp(h,30) or _yt_html(h,30))
 
 
291
  raw.extend(_fallback_shorts())
292
  seen=set();out=[]
293
  for v in raw:
294
  vid=v.get('id') or ''
295
  if not vid:
296
+ m=re.search(r'(?:v=|shorts/|youtu\.be/)([A-Za-z0-9_-]{11})',v.get('link',''));vid=m.group(1) if m else ''
297
+ title=_translate_title_vi(v.get('title') or 'YouTube Short');key=vid or re.sub(r'\W+','',title.lower())[:80]
 
 
298
  if not key or key in seen:continue
299
+ seen.add(key);item=dict(v);item['id']=vid;item['title']=title
300
+ if vid:item['link']='https://www.youtube.com/watch?v='+vid;item['img']='https://i.ytimg.com/vi/'+vid+'/hqdefault.jpg'
301
+ item['source']='yt';out.append(item)
 
 
 
 
302
  if len(out)>=40:break
303
  _SHORTS_CACHE_FINAL6.update({'t':now,'d':out})
304
  return JSONResponse(out)
305
 
306
  FINAL6_INJECT=r'''
307
  <style>
308
+ #ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4{display:none!important}.topic-final5{display:flex!important}.ai-wall-topic-live{margin:6px 4px;background:#1a1a1a;border:1px solid #2a2a2a;border-radius:8px;overflow:hidden}.hot-topic-row{display:flex;gap:6px;overflow-x:auto;padding:4px 0}.hot-chip{flex:0 0 auto;background:#222;border:1px solid #333;color:#ddd;border-radius:16px;padding:5px 10px;font-size:11px;cursor:pointer}.hot-chip:active{transform:scale(.96)}
309
  </style>
310
  <script>
311
  (function(){
312
  function esc(s){return String(s||'').replace(/[&<>"']/g,m=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[m]));}
313
  let liveTopicWall=[];
314
+ async function ensureHotTopics(){let inp=document.getElementById('ai-topic-input-final5');if(!inp||document.getElementById('hot-topic-row-final6'))return;let row=document.createElement('div');row.id='hot-topic-row-final6';row.className='hot-topic-row';row.innerHTML='<span style="color:#777;font-size:11px;padding:5px 0">Đang tải từ khóa nóng...</span>';inp.insertAdjacentElement('afterend',row);let j=await fetch('/api/hot_topics').then(r=>r.json()).catch(()=>({topics:[]}));let topics=j.topics||[];row.innerHTML=topics.slice(0,18).map(t=>`<button class="hot-chip" onclick="document.getElementById('ai-topic-input-final5').value='${esc(t.topic).replace(/'/g,'\\\'')}';document.getElementById('ai-topic-input-final5').focus();">${esc(t.label)}</button>`).join('')||'';}
315
+ async function ensureNewsShortsHome(){if(!document.getElementById('view-home')?.classList.contains('active'))return;let labels=[...document.querySelectorAll('.slider-wrap .slider-label')];let wraps=labels.filter(l=>/shorts|short /i.test(l.textContent||'')&&!/short ai/i.test(l.textContent||'')).map(l=>l.closest('.slider-wrap')).filter(Boolean);wraps.forEach((w,i)=>{if(i>0)w.remove();});let w=wraps[0];if(w){let seen=new Set();[...w.querySelectorAll('.slider-item')].forEach(it=>{let img=it.querySelector('img')?.src||'';let tt=(it.querySelector('.slider-title')?.textContent||'').trim().toLowerCase();let k=img||tt;if(k&&seen.has(k))it.remove();else if(k)seen.add(k);});if(w.querySelectorAll('.slider-item').length>=6)return;w.remove();}let sh=await fetch('/api/shorts?refresh=1').then(r=>r.json()).catch(()=>[]);if(!sh.length)return;let wrap=document.createElement('div');wrap.className='slider-wrap';wrap.id='shorts-final6-stable';let h='<div class="slider-header"><span class="slider-label">📱 Shorts Dân trí & SKĐS</span><span class="slider-note">Mới nhất</span></div><div class="slider-track">';sh.slice(0,30).forEach((a,i)=>{h+=`<div class="slider-item shorts-item" onclick="openTikTok('shorts',${i})"><div class="slider-thumb shorts-thumb">${a.img?`<img src="${esc(a.img)}">`:''}<div class="card-play">▶</div></div><div class="slider-title">${esc(a.title)}</div></div>`});h+='</div>';wrap.innerHTML=h;let comp=document.querySelector('.ai-compose')||document.getElementById('view-home').firstChild;if(comp)comp.after(wrap);else document.getElementById('view-home').prepend(wrap);}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
316
  function renderLiveTopicWall(){let home=document.getElementById('view-home');if(!home||!liveTopicWall.length)return;document.getElementById('ai-wall-topic-live')?.remove();let wrap=document.createElement('div');wrap.id='ai-wall-topic-live';wrap.className='ai-wall-topic-live';let h='<div class="slider-header"><span class="slider-label">🧱 Tường AI mới</span><span class="slider-note">Tổng hợp từ web</span></div><div class="slider-track">';liveTopicWall.slice(0,20).forEach((p,i)=>{h+=`<div class="wall-item"><div class="wall-thumb">${p.img?`<img src="${esc(p.img)}">`:''}</div><div class="wall-title">${esc(p.title)}</div><div class="wall-text">${esc(p.text)}</div><div class="wall-actions"><button class="primary" onclick="readLiveTopicWall(${i})">Xem</button></div></div>`});h+='</div>';wrap.innerHTML=h;let comp=document.querySelector('.ai-compose');if(comp)comp.after(wrap);else home.prepend(wrap);}
317
  window.readLiveTopicWall=function(i){let p=liveTopicWall[i];if(!p)return;showView('view-article');let imgs=(p.images||[]).filter(Boolean);let gal=imgs.length?'<div class="ai-wall-gallery">'+imgs.slice(0,12).map(u=>`<img src="${esc(u)}" loading="lazy">`).join('')+'</div>':(p.img?`<img class="article-img" src="${esc(p.img)}">`:'');document.getElementById('view-article').innerHTML=`<button class="back-btn" onclick="switchCat('home')">← Quay lại</button><div class="article-view"><span class="badge badge-ai">AI</span><h1 class="article-title">${esc(p.title)}</h1>${gal}<p class="article-p" style="white-space:pre-wrap">${esc(p.text)}</p><div class="article-actions"><button onclick="shareAI?shareAI(${JSON.stringify(p).replace(/"/g,'&quot;')},false):navigator.clipboard.writeText(location.href)">📤 Chia sẻ</button></div></div>`;window.scrollTo(0,0)};
318
  window.createTopicPostFinal5=async function(){let inp=document.getElementById('ai-topic-input-final5');let topic=(inp&&inp.value||'').trim();if(!topic)return alert('Nhập chủ đề trước');let btn=document.getElementById('ai-topic-btn-final5');if(btn){btn.disabled=true;btn.textContent='Đang crawl và tổng hợp...'}try{let r=await fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic})});let j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');liveTopicWall.unshift(j.post);if(inp)inp.value='';renderLiveTopicWall();readLiveTopicWall(0);alert('Đã tạo bài tổng hợp từ nội dung web và đăng lên Tường AI.');}catch(e){alert(e.message)}finally{if(btn){btn.disabled=false;btn.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen'}}};
319
+ setInterval(()=>{document.querySelectorAll('#ai-topic-input-final3,.topic-final3,#ai-topic-input-final4,.topic-final4').forEach(e=>(e.closest('.topic-final3,.topic-final4,.ai-compose-row')||e).remove());let b=document.getElementById('ai-topic-btn-final5');if(b){b.style.display='block';b.textContent='✨ Tạo bài tổng hợp từ web bằng Qwen';}ensureHotTopics();ensureNewsShortsHome();},1200);setTimeout(()=>{ensureHotTopics();ensureNewsShortsHome();},1200);
 
320
  })();
321
  </script>
322
  '''