Spaces:
Running
Running
| """Final6 runtime: fast homepage, fast shorts, Qwen topic with source details, hashtag sources, rewrite auto-title.""" | |
| import os, re, time, json, hashlib, asyncio, threading, requests | |
| from urllib.parse import urlparse, quote | |
| import ai_runtime_final5 as f5 | |
| from ai_runtime_final5 import app, base, rt, HTMLResponse, JSONResponse, Request, Query | |
| import html as html_lib | |
| SPACE_URL = "https://bep40-vnews.hf.space" | |
| SHORT_CHANNELS = ["baodantri7941", "baosuckhoedoisongboyte"] | |
| YOUTUBE_HANDLES = SHORT_CHANNELS | |
| DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data" | |
| os.makedirs(DATA_DIR, exist_ok=True) | |
| SHORTS_CACHE = {"t": 0, "d": []} | |
| AI_INTERACTIONS_FILE = os.path.join(DATA_DIR, "ai_interactions.json") | |
| SHORT_COMMENTS_FILE = os.path.join(DATA_DIR, "short_comments.json") | |
| def clean(s): | |
| return re.sub(r"\s+", " ", html_lib.unescape(s or "")).strip() | |
| def _domain(u): | |
| try: return urlparse(u or '').netloc.replace('www.', '') | |
| except: return '' | |
| def _lj(p, d): | |
| try: | |
| if os.path.exists(p): return json.load(open(p, 'r', encoding='utf-8')) | |
| except: pass | |
| return d | |
| def _sj(p, d): | |
| try: | |
| os.makedirs(os.path.dirname(p), exist_ok=True) | |
| open(p + '.tmp', 'w', encoding='utf-8').write(json.dumps(d, ensure_ascii=False)) | |
| os.replace(p + '.tmp', p) | |
| except: pass | |
| def _yt_ytdlp(handle, count=20): | |
| try: | |
| import yt_dlp | |
| url = f"https://www.youtube.com/@{handle}/shorts" | |
| opts = {'quiet': True, 'extract_flat': True, 'skip_download': True, 'playlist-end': count, 'ignoreerrors': True, 'no_warnings': True} | |
| with yt_dlp.YoutubeDL(opts) as ydl: | |
| info = ydl.extract_info(url, download=False) | |
| out = [] | |
| for e in (info or {}).get('entries') or []: | |
| vid = e.get('id') or '' | |
| if not re.match(r'^[A-Za-z0-9_-]{11}$', vid): continue | |
| title = e.get('title') or 'YouTube Short' | |
| out.append({'title': title, 'link': f'https://www.youtube.com/watch?v={vid}', 'img': f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg', 'source': 'yt', 'id': vid, 'channel': handle}) | |
| return out | |
| except: return [] | |
| def _yt_html(handle, count=20): | |
| try: | |
| r = requests.get(f"https://www.youtube.com/@{handle}/shorts", headers=getattr(base, 'HEADERS', {}), timeout=15) | |
| ids = []; out = [] | |
| for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"', r.text): | |
| vid = m.group(1) | |
| if vid in ids: continue | |
| ids.append(vid) | |
| snip = r.text[max(0, m.start()-1000):m.start()+1800] | |
| title = 'YouTube Short' | |
| mt = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', snip) or re.search(r'"accessibilityText":"([^"]+)"', snip) | |
| if mt: title = clean(mt.group(1).replace('\\n', ' ')) | |
| out.append({'title': title, 'link': f'https://www.youtube.com/watch?v={vid}', 'img': f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg', 'source': 'yt', 'id': vid, 'channel': handle}) | |
| if len(out) >= count: break | |
| return out | |
| except: return [] | |
| def _fallback_shorts(): | |
| out = []; seen = set() | |
| hard = [ | |
| ('Lu_iCQ5YwNM', 'Công an lập hồ sơ xử lý người phụ nữ chửi bới, tát tài xế ô tô | Dân trí', 'baodantri7941'), | |
| ('CwWvijF8BOA', 'Chú rể bật khóc nhận món quà bí mật người cha quá cố gửi 26 năm trước | Dân trí', 'baodantri7941'), | |
| ('7Pd6vZ2Lz1M', 'Hành động ấm lòng trong tìm kiếm học sinh tử vong ở sông Lô | SKĐS', 'baosuckhoedoisongboyte'), | |
| ('SlHLt_ZyPiE', 'Xử phạt người đàn ông xóa số điện thoại cứu hộ trên cao tốc Bắc - Nam | SKĐS', 'baosuckhoedoisongboyte'), | |
| ] | |
| for vid, title, ch in hard: | |
| if vid not in seen: | |
| seen.add(vid) | |
| out.append({'id': vid, 'title': title, 'channel': ch, 'link': f'https://www.youtube.com/watch?v={vid}', 'img': f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg', 'source': 'yt'}) | |
| return out | |
| def _fresh_shorts(): | |
| items = []; seen = set() | |
| for ch in YOUTUBE_HANDLES: | |
| got = _yt_ytdlp(ch, 24) or _yt_html(ch, 24) | |
| for v in got: | |
| if v['id'] not in seen: | |
| seen.add(v['id']); items.append(v) | |
| for v in _fallback_shorts(): | |
| if v['id'] not in seen: | |
| seen.add(v['id']); items.append(v) | |
| return items[:50] | |
| def _topic_image(topic): | |
| try: return base.pollinations_image_url(topic) | |
| except: return "https://image.pollinations.ai/prompt/" + quote("Vietnamese news editorial illustration " + topic) + "?width=1024&height=576&nologo=true" | |
| def _web_research_context(topic, limit=5): | |
| try: | |
| ctx, sources = base.web_context(topic, limit=limit) | |
| return {"context": ctx or "", "sources": sources or []} | |
| except: | |
| return {"context": "", "sources": []} | |
| def _fast_context(topic, limit=5): | |
| return _web_research_context(topic, limit) | |
| def _extract_source_details_from_context(ctx, sources): | |
| """Extract detailed source info from web context for source_details field.""" | |
| details = [] | |
| if not ctx or not sources: | |
| return details | |
| # Parse sources from context | |
| for s in sources[:8]: | |
| url = s.get('url', '') | |
| if not url: continue | |
| details.append({ | |
| 'title': s.get('title', ''), | |
| 'url': url, | |
| 'via': s.get('via', _domain(url)), | |
| 'content': s.get('excerpt', s.get('description', '')) | |
| }) | |
| return details | |
| _bg_home = {"t": 0, "d": []} | |
| _bg_shorts = {"t": 0, "d": []} | |
| _bg_lock = False | |
| def _bg(): | |
| global _bg_lock | |
| if _bg_lock: return | |
| _bg_lock = True | |
| try: | |
| # Fast homepage: just return empty, let frontend handle it | |
| _bg_home.update({"t": time.time(), "d": []}) | |
| # Shorts | |
| raw = [] | |
| for h in YOUTUBE_HANDLES: | |
| raw.extend(_yt_ytdlp(h, 20) or _yt_html(h, 20)) | |
| raw.extend(_fallback_shorts()) | |
| seen = set() | |
| out = [v for v in raw if v.get('id') and v['id'] not in seen and not seen.add(v['id'])] | |
| if out: _bg_shorts.update({"t": time.time(), "d": out[:40]}) | |
| except: pass | |
| finally: _bg_lock = False | |
| async def _s(): | |
| threading.Thread(target=_bg, daemon=True).start() | |
| threading.Thread(target=lambda: [time.sleep(600) or _bg() for _ in iter(int, 1)], daemon=True).start() | |
| # Remove endpoints to override | |
| app.router.routes = [r for r in app.router.routes if not ( | |
| getattr(r, 'path', None) in ('/api/homepage', '/api/shorts', '/api/ai_wall', '/api/topic_post', '/api/article/ask', '/api/topic/rewrite', '/api/rewrite_share', '/api/url_wall', '/api/short/comments', '/api/short/comment', '/api/storage_status', '/') and | |
| any(m in getattr(r, 'methods', set()) for m in ('GET', 'POST')) | |
| )] | |
| def _h(): | |
| n = time.time() | |
| if _bg_home['d']: | |
| if n - _bg_home['t'] > 300: threading.Thread(target=_bg, daemon=True).start() | |
| return JSONResponse(_bg_home['d']) | |
| threading.Thread(target=_bg, daemon=True).start() | |
| return JSONResponse([]) | |
| def _sh(refresh: int = Query(default=0)): | |
| n = time.time() | |
| if _bg_shorts['d'] and (not refresh or n - _bg_shorts['t'] < 120): | |
| if n - _bg_shorts['t'] > 600: threading.Thread(target=_bg, daemon=True).start() | |
| return JSONResponse(_bg_shorts['d']) | |
| data = _fresh_shorts() | |
| _bg_shorts.update({'t': n, 'd': data}) | |
| return JSONResponse(data) | |
| def _w(): | |
| n = int(time.time()) | |
| return JSONResponse({'posts': [p for p in f5.base._load_ai_wall() if n - int(p.get('ts') or 0) < 86400], 'persistent': os.path.isdir('/data')}) | |
| def _st(): | |
| return JSONResponse({'persistent': os.path.isdir('/data')}) | |
| def _gc(id: str = Query(...)): | |
| return JSONResponse({'comments': _lj(SHORT_COMMENTS_FILE, {}).get(id, [])}) | |
| async def _pc(request: Request): | |
| b = await request.json() | |
| v = str(b.get('id', '')).strip() | |
| t = clean(b.get('text', '')) | |
| if not v or not t: return JSONResponse({'error': 'missing'}, status_code=400) | |
| db = _lj(SHORT_COMMENTS_FILE, {}) | |
| c = db.get(v, []) | |
| c.insert(0, {'text': t[:300], 'ts': int(time.time())}) | |
| db[v] = c[:100] | |
| _sj(SHORT_COMMENTS_FILE, db) | |
| return JSONResponse({'comments': db[v]}) | |
| async def _ask(request: Request): | |
| b = await request.json() | |
| q = clean(b.get('question', '')) | |
| ctx = clean(b.get('context', '')) | |
| url = clean(b.get('url', '')) | |
| if not q: return JSONResponse({'error': 'missing question'}, status_code=400) | |
| title = ''; raw = '' | |
| if url: | |
| try: | |
| d = f5.base.scrape_any_url(url) | |
| title = d.get('title', '') | |
| raw = (d.get('summary', '') + '\n' + d.get('text', '')).strip() | |
| except: pass | |
| if not raw: raw = ctx[:12000] | |
| ans = await f5.base.qwen_generate( | |
| f'Bạn là VNEWS AI. Nội dung: "{title}"\n{raw[:9000]}\n\nHỏi: "{q}"\n\nTrả lời tự nhiên bằng tiếng Việt.', | |
| max_tokens=1200) | |
| return JSONResponse({'answer': ans or 'Chưa trả lời được.', 'title': title}) | |
| async def _rw(request: Request): | |
| b = await request.json() | |
| url = clean(b.get('url', '')) | |
| ctx = clean(b.get('context', '')) | |
| if not url.startswith('http'): return JSONResponse({'error': 'URL không hợp lệ'}, status_code=400) | |
| try: | |
| d = f5.base.scrape_any_url(url) | |
| title = d.get('title', '') | |
| raw = (d.get('summary', '') + '\n' + d.get('text', '')).strip() | |
| img = d.get('image') or '' | |
| except: | |
| title = ''; raw = ctx[:14000]; img = '' | |
| if len(raw) < 50: return JSONResponse({'error': 'Không đọc được bài'}, status_code=422) | |
| text = None | |
| try: | |
| text = await asyncio.wait_for(f5.base.qwen_generate( | |
| f'Tóm tắt đăng Tường AI:\nTiêu đề: {title}\n{raw[:14000]}\n\n4-6 ý chính. Cuối ghi nguồn.', | |
| image_url=img or None, max_tokens=1000), timeout=30) | |
| except: pass | |
| if not text or len(text) < 80: | |
| text = f"Tóm tắt: {title}\n\n{raw[:1200]}\n\nNguồn: {_domain(url)}" | |
| post = f5.base.make_post(title or 'Bài viết', text, img, url, 'rewrite', | |
| sources=[{'title': title, 'url': url, 'via': _domain(url)}]) | |
| ps = f5.base._load_ai_wall(); ps.insert(0, post); f5.base._save_ai_wall(ps) | |
| return JSONResponse({'post': post}) | |
| async def _tr(request: Request): | |
| b = await request.json() | |
| pid = str(b.get('post_id', '')).strip() | |
| if not pid: return JSONResponse({'error': 'missing post_id'}, status_code=400) | |
| ps = f5.base._load_ai_wall() | |
| p = next((x for x in ps if str(x.get('id')) == pid), None) | |
| if not p: return JSONResponse({'error': 'Bài không tồn tại'}, status_code=404) | |
| urls = list(dict.fromkeys( | |
| [s['url'] for s in (p.get('source_details') or []) if s.get('url')] + | |
| [s['url'] for s in (p.get('sources') or []) if s.get('url')] | |
| ))[:5] | |
| parts = [] | |
| for u in urls: | |
| try: | |
| d = f5.base.scrape_any_url(u) | |
| t = d.get('title', '') | |
| r = (d.get('summary', '') + '\n' + d.get('text', '')).strip() | |
| if r and len(r) > 150: | |
| parts.append(f"[{_domain(u)}] {t}\n{r}") | |
| except: pass | |
| ac = '\n---\n'.join(parts) if parts else (p.get('text') or '') | |
| title = p.get('title', '') | |
| text = None | |
| try: | |
| text = await asyncio.wait_for(f5.base.qwen_generate( | |
| f'Viết lại:\nChủ đề: {title}\n{ac[:16000]}\n\nTiêu đề mới + 4-6 ý + nguồn.', | |
| image_url=p.get('img'), max_tokens=1200), timeout=35) | |
| except: pass | |
| if not text or len(text) < 100: | |
| text = f"Tóm tắt: {title}\n\n{ac[:1500]}\n\nNguồn: VNEWS AI" | |
| np = f5.base.make_post('Rewrite: ' + title, text, p.get('img', ''), '', 'rewrite_topic', sources=p.get('sources', [])) | |
| np['images'] = p.get('images', []) | |
| all_p = f5.base._load_ai_wall(); all_p.insert(0, np); f5.base._save_ai_wall(all_p) | |
| return JSONResponse({'post': np}) | |
| async def _tp(request: Request): | |
| b = await request.json() | |
| topic = clean(b.get('topic', '')) | |
| if not topic: return JSONResponse({'error': 'missing topic'}, status_code=400) | |
| img = _topic_image(topic) | |
| research = _fast_context(topic) | |
| ctx = research.get('context', '') | |
| src = research.get('sources', []) | |
| det = _extract_source_details_from_context(ctx, src) | |
| if not ctx or not src: | |
| return JSONResponse({'error': 'Không tìm được nội dung.'}, status_code=422) | |
| sb = '\n\n'.join([f"[{i+1}] {d.get('title', '')} ({d.get('via', '')})\n{d.get('content', '')[:1400]}" for i, d in enumerate(det)]) if det else ctx[:18000] | |
| text = None | |
| try: | |
| text = await asyncio.wait_for(f5.base.qwen_generate( | |
| f'Viết bài tiếng Việt VỀ: "{topic}"\nNGUỒN:\n{sb[:18000]}\nCHỈ viết về "{topic}". 5-8 đoạn. Cuối có nguồn.', | |
| image_url=img, max_tokens=1700), timeout=35) | |
| except: pass | |
| if not text or len(text) < 300: | |
| text = f"{topic}: tổng hợp\n\n" + '\n'.join([f"• {d['title']}: {d.get('content', '')[:300]}" for d in (det or [])[:6]]) + "\n\nNguồn: " + ', '.join(sorted({d.get('via', '') for d in (det or []) if d.get('via')})) | |
| post = f5.base.make_post(topic, text, img, '', 'topic_focused', sources=[s for s in src if s.get('url')]) | |
| post['images'] = [img]; post['source_details'] = det | |
| ps = f5.base._load_ai_wall(); ps.insert(0, post); f5.base._save_ai_wall(ps) | |
| return JSONResponse({'post': post}) | |
| FINAL6_INJECT = r''' | |
| <style> | |
| /* Livescore */ | |
| .ls-content{max-height:480px;overflow-y:auto;padding:0 6px 8px;font-size:12px;color:#ddd} | |
| .ls-content ul{list-style:none;padding:0;margin:0} | |
| .ls-content .title-content{display:flex;gap:6px;align-items:center;background:#222;border-radius:4px;margin:4px 0;padding:5px 8px} | |
| .ls-content .title-content img{width:18px;height:18px} | |
| .ls-content .title-content strong{font-size:11px;color:#ccc} | |
| .ls-content .match-detail{padding:6px;border-bottom:1px solid #262626;cursor:pointer} | |
| .ls-content .match-detail:hover{background:#1a2a1f} | |
| .ls-content .match{display:flex;flex-wrap:wrap;align-items:center;gap:4px} | |
| .ls-content .datetime{width:100%;font-size:9px;color:#888} | |
| .ls-content .teams{display:flex;width:100%;align-items:center;gap:4px} | |
| .ls-content .team{flex:1;display:flex;align-items:center;gap:4px;min-width:0} | |
| .ls-content .team .name{font-size:11px;color:#ddd;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} | |
| .ls-content .team .logo img{width:18px;height:18px} | |
| .ls-content .home-team{justify-content:flex-end;text-align:right} | |
| .ls-content .status{flex:0 0 54px;text-align:center} | |
| .ls-content .status a{color:#fff;text-decoration:none;font-weight:800;font-size:12px} | |
| .ls-content .status .label{font-size:8px;color:#888;display:block} | |
| .ls-content .status .label.live{color:#e74c3c} | |
| .ls-content .info,.ls-content .btns{display:none} | |
| </style> | |
| <script> | |
| (function(){ | |
| function esc(s){return String(s||'').replace(/[&<>"']/g,m=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[m]));} | |
| // Block slow YouTube refresh on first load | |
| var _origFetch=window.fetch,_allowRefresh=false; | |
| window.fetch=function(url,opts){try{if(String(url).indexOf('/api/shorts?refresh=1')>-1&&!_allowRefresh)url='/api/shorts';}catch(e){}return _origFetch.call(this,url,opts);}; | |
| setTimeout(function(){_allowRefresh=true;},8000); | |
| })(); | |
| </script> | |
| ''' | |
| FINAL6_FAST_HOME_INJECT = r''' | |
| <style> | |
| .storage-warn{background:#332200;border:1px solid #664400;color:#ffcc00;padding:8px 12px;border-radius:8px;font-size:11px;margin:6px 4px} | |
| </style> | |
| <script> | |
| (function(){ | |
| fetch('/api/storage_status').then(function(r){return r.json()}).then(function(j){ | |
| if(!j.persistent){var h=document.getElementById('view-home');if(h){var w=document.createElement('div');w.className='storage-warn';w.innerHTML='⚠️ <b>Persistent Storage chưa bật.</b> Bật: Space Settings → Persistent Storage → Small.';h.prepend(w);}} | |
| }); | |
| })(); | |
| </script> | |
| ''' | |
| FINAL6E_INJECT = r''' | |
| <style> | |
| /* Kill ALL duplicate slides/walls from old layers */ | |
| #ai-short-home,.ai-short-home,.ai-short-card-final,[id*="ai-shorts-patched"]{display:none!important} | |
| </style> | |
| <script> | |
| (function(){ | |
| // Kill old renderers | |
| window.renderTopicWallE=function(){}; | |
| window.renderAIShortHome=function(){}; | |
| window.renderAIShorts7=function(){}; | |
| window.renderPatchedWall=function(){}; | |
| window.renderAiShorts=function(){}; | |
| window.renderWall=function(){}; | |
| window.renderAIShorts=function(){}; | |
| window.loadPatchedWall=function(){}; | |
| window.refreshFinalWall3=function(){}; | |
| // Remove duplicate slides | |
| setInterval(function(){ | |
| document.querySelectorAll('#ai-short-home,.ai-short-home,[id*="ai-shorts-patched"]').forEach(function(el){el.remove()}); | |
| },2000); | |
| })(); | |
| </script> | |
| ''' | |
| async def _index(): | |
| html = f5.f4.f3.f2.f1._load_index_html() | |
| body = '' | |
| body += getattr(rt.old, 'PATCH_INJECT', '') | |
| body += f5.f4.f3.f2.f1.FINAL_INJECT + f5.f4.f3.FINAL3_INJECT + f5.f4.FINAL4_INJECT + f5.FINAL5_INJECT | |
| body += FINAL6_INJECT + FINAL6_FAST_HOME_INJECT + FINAL6E_INJECT | |
| return HTMLResponse(html.replace('</body>', body + '\n</body>') if '</body>' in html else html + body) | |