Limit topics to requested VN sources and speed homepage

ai_runtime_final6.py

@@ -1141,3 +1141,185 @@ def _candidate_urls(topic):
                     seen.add(it['url']);items.append(it)
             if len(items)>=12:break
     return items[:20]
+
+
+# ===== FINAL6G: SOURCE-LIMITED FAST TOPICS AND FAST HOME =====
+# Limit hot hashtags/topic context to requested sources and make homepage APIs return quickly.
+_SOURCE_FEEDS = [
+    ('VnExpress','https://vnexpress.net/rss/tin-moi-nhat.rss','vnexpress.net'),
+    ('VnExpress Thời sự','https://vnexpress.net/rss/thoi-su.rss','vnexpress.net'),
+    ('VnExpress Thế giới','https://vnexpress.net/rss/the-gioi.rss','vnexpress.net'),
+    ('VnExpress Kinh doanh','https://vnexpress.net/rss/kinh-doanh.rss','vnexpress.net'),
+    ('VnExpress Công nghệ','https://vnexpress.net/rss/so-hoa.rss','vnexpress.net'),
+    ('VnExpress Thể thao','https://vnexpress.net/rss/the-thao.rss','vnexpress.net'),
+    ('Dân trí','https://dantri.com.vn/rss/home.rss','dantri.com.vn'),
+    ('Dân trí Xã hội','https://dantri.com.vn/rss/xa-hoi.rss','dantri.com.vn'),
+    ('Dân trí Kinh doanh','https://dantri.com.vn/rss/kinh-doanh.rss','dantri.com.vn'),
+    ('Dân trí Thể thao','https://dantri.com.vn/rss/the-thao.rss','dantri.com.vn'),
+    ('Dân trí Công nghệ','https://dantri.com.vn/rss/suc-manh-so.rss','dantri.com.vn'),
+    ('VTV','https://vtv.vn/rss/trang-chu.rss','vtv.vn'),
+    ('VTV Thời sự','https://vtv.vn/rss/thoi-su.rss','vtv.vn'),
+    ('GenK','https://genk.vn/rss/home.rss','genk.vn'),
+    ('GenK AI','https://genk.vn/ai.rss','genk.vn'),
+    ('VnReview','https://vnreview.vn/rss/tin-moi-nhat.rss','vnreview.vn'),
+    ('VnReview Công nghệ','https://vnreview.vn/rss/cong-nghe.rss','vnreview.vn'),
+    ('Vật Vờ Studio','https://vatvostudio.vn/feed/','vatvostudio.vn'),
+    ('Thể thao văn hóa','https://thethaovanhoa.vn/rss/home.rss','thethaovanhoa.vn'),
+    ('Thể thao văn hóa World Cup','https://thethaovanhoa.vn/rss/world-cup-2026.rss','thethaovanhoa.vn'),
+]
+_SOURCE_CACHE={'t':0,'items':[]}
+_FAST_ROUTE_CACHE={}
+
+def _feed_items_source_limited(max_per_feed=10):
+    now=time.time()
+    if _SOURCE_CACHE['items'] and now-_SOURCE_CACHE['t']<600:return _SOURCE_CACHE['items']
+    items=[];seen=set()
+    def one(feed):
+        name,url,dom=feed;out=[]
+        try:
+            r=requests.get(url,headers=UA,timeout=3.5);r.encoding='utf-8'
+            soup=BeautifulSoup(r.text,'xml')
+            for it in soup.find_all('item')[:max_per_feed*2]:
+                title=clean(it.find('title').get_text(' ',strip=True) if it.find('title') else '')
+                link=clean(it.find('link').get_text(strip=True) if it.find('link') else '')
+                desc_raw=it.find('description').get_text(' ',strip=True) if it.find('description') else ''
+                ds=BeautifulSoup(desc_raw,'lxml')
+                img='';im=ds.find('img')
+                if im:img=im.get('src','') or im.get('data-src','')
+                desc=clean(ds.get_text(' ',strip=True))[:700]
+                if title and link:
+                    out.append({'title':title,'url':link,'link':link,'source':name,'via':name,'domain':dom,'snippet':desc,'img':img})
+                if len(out)>=max_per_feed:break
+        except Exception:pass
+        return out
+    try:
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        with ThreadPoolExecutor(max_workers=8) as ex:
+            futs=[ex.submit(one,f) for f in _SOURCE_FEEDS]
+            for f in as_completed(futs,timeout=5.5):
+                try:
+                    for it in f.result() or []:
+                        if it['url'] not in seen:
+                            seen.add(it['url']);items.append(it)
+                except Exception:pass
+    except Exception:
+        for f in _SOURCE_FEEDS[:8]:
+            for it in one(f):
+                if it['url'] not in seen:
+                    seen.add(it['url']);items.append(it)
+    _SOURCE_CACHE.update({'t':now,'items':items})
+    return items
+
+def _score_topic_source(topic,it):
+    toks=[w.lower() for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',topic or '') if len(w)>1 and w.lower() not in STOP_WORDS]
+    hay=(it.get('title','')+' '+it.get('snippet','')+' '+it.get('source','')).lower()
+    if not toks:return 0
+    score=sum((3 if len(t)>3 else 1) for t in toks if t in hay)
+    if topic.lower().strip() and topic.lower().strip() in hay:score+=12
+    return score
+
+def _hot_topics():
+    now=time.time()
+    if _HOT_CACHE['d'] and now-_HOT_CACHE['t']<600:return _HOT_CACHE['d']
+    freq={};display={}
+    for it in _feed_items_source_limited(8)[:180]:
+        title=re.sub(r'\s+-\s+.*$','',it.get('title',''))
+        kws=[]
+        for m in re.findall(r'([A-ZĐÀ-Ỹ][A-Za-zÀ-ỹ0-9]+(?:\s+[A-ZĐÀ-ỸA-Za-zÀ-ỹ0-9][A-Za-zÀ-ỹ0-9]+){1,4})',title):
+            if len(m)>=6:kws.append(m)
+        kws += _keywords_from_title(title)
+        for kw in kws[:4]:
+            words=[w for w in clean(kw).split() if w.lower() not in STOP_WORDS]
+            if len(words)<2:continue
+            kw=' '.join(words[:5])
+            if 6<=len(kw)<=55:
+                key=kw.lower();freq[key]=freq.get(key,0)+1;display[key]=kw
+    topics=[]
+    for key,_ in sorted(freq.items(),key=lambda x:x[1],reverse=True):
+        kw=display[key];topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
+        if len(topics)>=24:break
+    for kw in ['Giá vàng trong nước','AI tại Việt Nam','Bóng đá Việt Nam','Kinh tế Việt Nam','Công nghệ AI','Vật Vờ Studio','World Cup 2026','Sức khỏe cộng đồng']:
+        if not any(t['topic'].lower()==kw.lower() for t in topics):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
+    _HOT_CACHE.update({'t':now,'d':topics[:24]})
+    return _HOT_CACHE['d']
+
+def _fast_context(topic):
+    now=time.time();key='source_limited_ctx:'+topic.lower().strip()
+    if key in _FAST_TOPIC_CACHE and now-_FAST_TOPIC_CACHE[key]['t']<600:return _FAST_TOPIC_CACHE[key]['d']
+    pool=_feed_items_source_limited(12)
+    scored=[]
+    for it in pool:
+        sc=_score_topic_source(topic,it)
+        if sc>0:scored.append((sc,it))
+    if not scored:
+        # Try broader matching by first token only before giving up.
+        first=(_fast_topic_tokens(topic) or [''])[0]
+        if first:
+            for it in pool:
+                if first in (it.get('title','')+' '+it.get('snippet','')).lower():scored.append((1,it))
+    picked=[it for sc,it in sorted(scored,key=lambda x:(x[0],len(x[1].get('snippet',''))),reverse=True)[:8]]
+    if not picked:picked=pool[:6]
+    blocks=[];src=[]
+    for it in picked:
+        content=it.get('snippet','') or it.get('title','')
+        blocks.append(f"NGUỒN: {it.get('source','')}\nTIÊU ĐỀ: {it.get('title','')}\nTÓM TẮT RSS:\n{content}")
+        src.append({'title':it.get('title',''),'url':it.get('url',''),'via':it.get('source',''),'snippet':content})
+    data={'context':'\n\n---\n\n'.join(blocks),'sources':src,'count':len(blocks)}
+    _FAST_TOPIC_CACHE[key]={'t':now,'d':data}
+    return data
+
+# Override slow search functions to never crawl open web during topic generation.
+def _web_research_context(topic):
+    return _fast_context(topic)
+
+def _candidate_urls(topic):
+    return _fast_context(topic).get('sources',[])
+
+# Fast homepage endpoints from requested source RSS; no slow HTML scrapers.
+def _fast_homepage_sources():
+    now=time.time();key='home_sources'
+    if key in _FAST_ROUTE_CACHE and now-_FAST_ROUTE_CACHE[key]['t']<600:return _FAST_ROUTE_CACHE[key]['d']
+    groups=[];seen=set()
+    group_map=[('Tin mới','https://vnexpress.net/rss/tin-moi-nhat.rss','vne'),('Thời Sự','https://vnexpress.net/rss/thoi-su.rss','vne'),('Kinh Doanh','https://vnexpress.net/rss/kinh-doanh.rss','vne'),('Công Nghệ','https://vnexpress.net/rss/so-hoa.rss','vne'),('Dân Trí','https://dantri.com.vn/rss/home.rss','dantri'),('GenK','https://genk.vn/rss/home.rss','genk'),('VnReview','https://vnreview.vn/rss/tin-moi-nhat.rss','vnreview')]
+    for g,u,s in group_map:
+        for it in _rss_articles_fast(u,g,s,6) if '_rss_articles_fast' in globals() else []:
+            if it['link'] not in seen:
+                seen.add(it['link']);groups.append(it)
+    _FAST_ROUTE_CACHE[key]={'t':now,'d':groups}
+    return groups
+
+for _p in ['/api/homepage','/api/dantri_hot','/api/vne_video','/api/highlights','/api/hot_topics','/api/topic_sources']:
+    app.router.routes=[r for r in app.router.routes if not (getattr(r,'path',None)==_p and 'GET' in getattr(r,'methods',set()))]
+@app.get('/api/homepage')
+def api_homepage_source_fast():return JSONResponse(_fast_homepage_sources())
+@app.get('/api/dantri_hot')
+def api_dantri_hot_source_fast():
+    data=[{**it,'source':'dantri','link':it.get('url') or it.get('link')} for it in _feed_items_source_limited(8) if it.get('domain')=='dantri.com.vn'][:12]
+    return JSONResponse(data)
+@app.get('/api/vne_video')
+def api_vne_video_source_fast():
+    return JSONResponse([])  # do not block homepage if VnEgo is slow
+@app.get('/api/highlights')
+def api_highlights_source_fast():return JSONResponse([])
+@app.get('/api/hot_topics')
+def api_hot_topics_source_fast():return JSONResponse({'topics':_hot_topics(),'sources':'vn_only'})
+@app.get('/api/topic_sources')
+def api_topic_sources_source_fast(topic:str=Query(...)):
+    data=_fast_context(clean(topic));return JSONResponse({'count':data.get('count',0),'sources':data.get('sources',[]),'has_context':bool(data.get('context')),'mode':'source_limited_rss'})
+
+# Override root: include all existing UI but add a script that prevents forced shorts refresh on initial load.
+ROOT_FAST_INJECT="""
+<script>
+(function(){
+const oldFetch=window.fetch;window.__allowShortRefresh=false;
+window.fetch=function(url,opts){try{let u=String(url||'');if(u.includes('/api/shorts?refresh=1')&&!window.__allowShortRefresh)url='/api/shorts';}catch(e){}return oldFetch.call(this,url,opts)};
+setTimeout(()=>{window.__allowShortRefresh=true;},8000);
+})();
+</script>
+"""
+app.router.routes=[r for r in app.router.routes if not (getattr(r,'path',None)=='/' and 'GET' in getattr(r,'methods',set()))]
+@app.get('/')
+async def index_final_fast_sources():
+    html=f5.f4.f3.f2.f1._load_index_html()
+    body=getattr(rt.old,'PATCH_INJECT','')+f5.f4.f3.f2.f1.FINAL_INJECT+f5.f4.f3.FINAL3_INJECT+f5.f4.FINAL4_INJECT+f5.FINAL5_INJECT+FINAL6_INJECT+globals().get('FINAL6_FAST_HOME_INJECT','')+globals().get('FINAL6E_INJECT','')+globals().get('FINAL6F_INJECT','')+ROOT_FAST_INJECT
+    return HTMLResponse(html.replace('</body>',body+'\n</body>') if '</body>' in html else html+body)