"""VNEWS v2 Entry Point - with fast bongda proxy + rewrite endpoints + multilingual TTS""" import sys, os from main import app, HEADERS, BONGDA_HEADERS, fetch_bongda_api, HL_LEAGUES try: import ai_ext except Exception as e: print(f"[WARN] ai_ext import failed: {e}") try: import ai_patch except Exception as e: print(f"[WARN] ai_patch import failed: {e}") from fastapi.responses import HTMLResponse, JSONResponse, FileResponse, Response from fastapi.staticfiles import StaticFiles from starlette.routing import Mount from fastapi import Query, Request, UploadFile, File, Form import requests as req from bs4 import BeautifulSoup import re, html as html_lib, json, threading, time, uuid from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import quote import asyncio HL_LEAGUES['friendly'] = {"path": "giai-khac/friendly", "name": "Giao hữu", "emoji": "🤝"} STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static') SPACE = "https://bep40-vnews.hf.space" # SEO URL base for share links app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/' and hasattr(r,'methods') and 'GET' in getattr(r,'methods',set()))] app.routes[:]=[r for r in app.routes if not isinstance(r, Mount)] app.router.routes=[r for r in app.router.routes if not isinstance(r, Mount)] def _clean(s): return re.sub(r"\s+"," ",html_lib.unescape(str(s or""))).strip() # Cache for match details (5 min TTL) _match_cache = {} # === FAST BONGDA PROXY ENDPOINT === def _get_match_detail(event_id, slug=None): headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "text/html", "Referer": "https://bongda.com.vn/"} if slug: url = f"https://bongda.com.vn/tran-dau/{event_id}/centre/{slug}" else: url = f"https://bongda.com.vn/tran-dau/{event_id}" resp = req.get(url, headers=headers, timeout=15, allow_redirects=True) if resp.status_code != 200: return None soup = BeautifulSoup(resp.text, 'html.parser') result = {"event_id": event_id, "found": False, "sections": []} info = {} tel = soup.select_one('.teams') if tel: he = tel.select_one('.team.home') if he: p_tags = [p for p in he.select('p') if not p.get('class') or 'logo' not in p.get('class', [])] if p_tags: info['home_team'] = _clean(p_tags[0].get_text()) lo = he.select_one('img') if lo: info['home_logo'] = lo.get('src', '') ae = tel.select_one('.team.away') if ae: p_tags = ae.select('p') team_ps = [p for p in p_tags if not p.get('class') or 'logo' not in p.get('class', [])] if team_ps: info['away_team'] = _clean(team_ps[-1].get_text()) lo = ae.select_one('img') if lo: info['away_logo'] = lo.get('src', '') sc = tel.select_one('.score') if sc: parts = [_clean(p.get_text()) for p in sc.select('p')] if len(parts) >= 2: info['score'] = f"{parts[0]} - {parts[1]}" lb = sc.select_one('.label') if lb: info['status_label'] = _clean(lb.get_text()) if info.get('home_team') and info.get('away_team'): result['info'] = info result['found'] = True result['sections'].append('info') events = [] for ev in soup.select('.events .period .event'): ev_cls = ' '.join(ev.get('class', [])) ev_data = {'team': 'home' if 'home' in ev_cls else 'away', 'period': '', 'type': 'unknown', 'time': '', 'players': ''} parent = ev.parent if parent: h2 = parent.find('h2') if h2: ev_data['period'] = _clean(h2.get_text()) if ev.select_one('[class*="goal"]'): ev_data['type'] = 'goal' elif ev.select_one('[class*="redcard"]'): ev_data['type'] = 'redcard' elif ev.select_one('[class*="yellowcard"]'): ev_data['type'] = 'yellowcard' elif ev.select_one('[class*="substitution"]'): ev_data['type'] = 'substitution' players_el = ev.select_one('.players') if players_el: pl_text = _clean(players_el.get_text(' ', strip=True)) m = re.match(r"(\d+)'(.*)", pl_text) if m: ev_data['time'] = f"{m.group(1)}'" ev_data['players'] = m.group(2) else: ev_data['players'] = pl_text events.append(ev_data) if events: result['events'] = events result['sections'].append('events') pred = soup.select_one('.prediction-card') if pred: team_info = pred.select_one('.team-info') if team_info: teams = team_info.select('.team') pred_data = {} if len(teams) >= 2: pred_data['home_name'] = _clean(teams[0].select_one('.team-name').get_text()) if teams[0].select_one('.team-name') else '' pred_data['away_name'] = _clean(teams[1].select_one('.team-name').get_text()) if teams[1].select_one('.team-name') else '' divider = team_info.select_one('.divider') if divider: pred_data['result'] = _clean(divider.get_text()) vc = pred.select_one('.vote-count') if vc: pred_data['vote_count'] = _clean(vc.get_text()) result['prediction'] = pred_data recent = [] ml = soup.select_one('.matches-list') if ml: for item in ml.select('.match-detail, .match-item, li'): de = item.select_one('.date, .time') le = item.select_one('.league') he_item = item.select_one('.home, .team-home') ae_item = item.select_one('.away, .team-away') se = item.select_one('.score, .result') if he_item or ae_item: recent.append({'date': _clean(de.get_text()) if de else '', 'league': _clean(le.get_text()) if le else '', 'home': _clean(he_item.get_text()) if he_item else '', 'away': _clean(ae_item.get_text()) if ae_item else '', 'score': _clean(se.get_text()) if se else 'vs'}) if recent: result['recent_matches'] = recent result['sections'].append('recent') try: api_h = {"User-Agent": "Mozilla/5.0", "Accept": "application/json", "X-Requested-With": "XMLHttpRequest", "Referer": "https://bongda.com.vn/"} ar = req.get(f"https://bongda.com.vn/api/fixtures/h2h-stats?event_id={event_id}", headers=api_h, timeout=10) if ar.status_code == 200: ad = ar.json() if ad.get('status') == 'success' and ad.get('html'): asp = BeautifulSoup(ad['html'], 'html.parser') ast = {} for row in asp.select('li, tr'): cells = row.select('td, span, p') if len(cells) >= 3: lb = _clean(cells[0].get_text()) if lb: ast[lb] = {'home': _clean(cells[1].get_text()), 'away': _clean(cells[2].get_text())} if ast: result['h2h_stats_parsed'] = ast result['sections'].append('h2h_stats') except: pass return result @app.get('/api/proxy/bongda') def proxy_bongda(event_id: int = Query(default=None), slug: str = Query(default=None)): if event_id is None: return JSONResponse({'error': 'event_id required'}, status_code=400) cache_key = f"{event_id}_{slug}" now = time.time() cached = _match_cache.get(cache_key) if cached and now - cached.get('_ts', 0) < 300: return JSONResponse(cached) try: result = _get_match_detail(event_id, slug) if result: result['_ts'] = now _match_cache[cache_key] = result return JSONResponse(result) except Exception as e: err = {"event_id": event_id, "found": False, "error": str(e), "_ts": now} _match_cache[cache_key] = err return JSONResponse(err) return JSONResponse({"event_id": event_id, "found": False}) @app.get('/api/match/{event_id}/detail') def api_match_detail(event_id: int, url: str = Query(default=None)): slug = None if url: m = re.match(r'.+/tran-dau/\d+/(?:centre|preview)/(.+)', url) if m: slug = m.group(1) cache_key = f"{event_id}_{slug or ''}" now = time.time() cached = _match_cache.get(cache_key) if cached and now - cached.get('_ts', 0) < 300: return JSONResponse(cached) try: if not slug: try: home_r = req.get("https://bongda.com.vn/", headers={"User-Agent": "Mozilla/5.0"}, timeout=10) if home_r.status_code == 200: home_soup = BeautifulSoup(home_r.text, 'html.parser') for a in home_soup.select(f'a[href*="/tran-dau/{event_id}/"]'): href = a.get('href', '') m = re.match(r'/tran-dau/\d+/(?:centre|preview)/(.+)', href) if m: slug = m.group(1) cache_key = f"{event_id}_{slug}" break except: pass result = _get_match_detail(event_id, slug) if result: result['_ts'] = now _match_cache[cache_key] = result return JSONResponse(result) except Exception as e: err = {"event_id": event_id, "found": False, "error": str(e), "_ts": now} _match_cache[cache_key] = err return JSONResponse(err) return JSONResponse({"event_id": event_id, "found": False}) _STOP=set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì này đã để'.split()) def _has_kw(topic,title): tl=topic.lower();tt=(title or'').lower() if tl in tt:return True words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',tl) if len(w)>2 and w not in _STOP] if not words:return True return any(w in tt for w in words) def _s_vnexpress(topic,limit=8): items=[] try: r=req.get(f"https://timkiem.vnexpress.net/?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml') for art in soup.select('article.item-news')[:limit]: a=art.select_one('h2 a, h3 a') if a and a.get('href'): t=_clean(a.get('title','') or a.get_text(strip=True)) if _has_kw(topic,t):items.append({'title':t,'url':a['href'],'via':'VnExpress'}) except:pass return items def _s_dantri(topic,limit=8): items=[] try: r=req.get(f"https://dantri.com.vn/tim-kiem/{quote(topic)}.htm",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('h3 a[href], .article-title a[href]')[:limit*2]: t=_clean(a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if not href.startswith('http'):href='https://dantri.com.vn'+href items.append({'title':t,'url':href,'via':'Dân Trí'}) if len(items)>=limit:break except:pass return items def _s_vietnamnet(topic,limit=6): items=[] try: r=req.get(f"https://vietnamnet.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('h3 a[href], .vnn-title a')[:limit*2]: t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if not href.startswith('http'):href='https://vietnamnet.vn'+href items.append({'title':t,'url':href,'via':'VietNamNet'}) if len(items)>=limit:break except:pass return items def _s_bongda(topic,limit=5): items=[] try: r=req.get(f"https://bongda.com.vn/tim-kiem.html?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('h3 a[href], .title a[href]')[:limit*2]: t=_clean(a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if not href.startswith('http'):href='https://bongda.com.vn'+href items.append({'title':t,'url':href,'via':'Bóng Đá'}) if len(items)>=limit:break except:pass return items def _s_genk(topic,limit=5): items=[] try: r=req.get(f"https://genk.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('a[href$=".chn"]')[:limit*3]: t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if href.startswith('/'):href='https://genk.vn'+href items.append({'title':t,'url':href,'via':'GenK'}) if len(items)>=limit:break except:pass return items def _s_thanhnien(topic,limit=6): items=[] try: r=req.get(f"https://thanhnien.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('h3 a[href], .box-title a')[:limit*2]: t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if not href.startswith('http'):href='https://thanhnien.vn'+href items.append({'title':t,'url':href,'via':'Thanh Niên'}) if len(items)>=limit:break except:pass return items def _s_tuoitre(topic,limit=6): items=[] try: r=req.get(f"https://tuoitre.vn/tim-kiem.htm?keywords={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('h3 a[href], .box-title-text a')[:limit*2]: t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if not href.startswith('http'):href='https://tuoitre.vn'+href items.append({'title':t,'url':href,'via':'Tuổi Trẻ'}) if len(items)>=limit:break except:pass return items def _s_thethaovanhoa(topic,limit=5): items=[] try: r=req.get(f"https://thethaovanhoa.vn/tim-kiem.htm?keyword={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml') for a in soup.select('h3 a[href], .title a[href]')[:limit*2]: t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','') if t and len(t)>15 and _has_kw(topic,t): if not href.startswith('http'):href='https://thethaovanhoa.vn'+href items.append({'title':t,'url':href,'via':'TT&VH'}) if len(items)>=limit:break except:pass return items def _search_all(topic,limit=36): results={} with ThreadPoolExecutor(8) as ex: futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,5):'bd',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'} for f in as_completed(futs,timeout=14): try:results[futs[f]]=f.result() except:results[futs[f]]=[] srcs=list(results.values());out=[];seen=set() for i in range(max((len(s) for s in srcs),default=0)): for s in srcs: if i= 2: seen_imgs = set() for child in el.find_all(['p','h2','h3','figure','img'], recursive=True): if child.name == 'p': t = child.get_text(strip=True) if t and len(t) > 15: body.append({'type': 'p', 'text': t}) elif child.name in ('h2','h3'): t = child.get_text(strip=True) if t: body.append({'type': 'heading', 'text': t}) elif child.name in ('figure','img'): im = child if child.name == 'img' else child.find('img') if im: src = im.get('data-src') or im.get('src') or im.get('data-lazy') or '' if src and 'base64' not in src and src not in seen_imgs: seen_imgs.add(src) if src.startswith('//'): src = 'https:' + src body.append({'type': 'img', 'src': src}) if child.name == 'figure': cap = child.find('figcaption') if cap: ct = cap.get_text(strip=True) if ct: body.append({'type': 'p', 'text': ct}) if len(body) >= 2: return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img, 'body': body[:50], 'source': domain, 'url': url} if title and (summary or og_img): fallback = [] if og_img: fallback.append({'type': 'img', 'src': og_img}) if summary: fallback.append({'type': 'p', 'text': summary}) if fallback: return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img, 'body': fallback, 'source': domain, 'url': url, 'fallback': True} if title: return {'title': _clean(title), 'summary': '', 'og_image': '', 'body': [{'type': 'p', 'text': 'Nội dung đang được tải...'}], 'source': domain, 'url': url, 'fallback': True} break except Exception: continue return None @app.get('/api/article') def api_article_v2(url: str = Query(...)): from urllib.parse import unquote safe_url = unquote(url) try: now = time.time() cached = _article_cache.get(safe_url) if cached and now - cached['t'] < _article_cache_ttl: resp = JSONResponse(cached['d']) resp.headers["Cache-Control"] = "public, max-age=1800" return resp data = _scrape_article_fast(safe_url) if data and data.get('body'): _article_cache[safe_url] = {'d': data, 't': now} resp = JSONResponse(data) resp.headers["Cache-Control"] = "public, max-age=1800" return resp result = {'error': 'Không đọc được', 'url': safe_url} resp = JSONResponse(result) resp.headers["Cache-Control"] = "public, max-age=60" return resp except Exception as e: return JSONResponse({'error': f'Server error: {str(e)[:100]}', 'url': safe_url}, status_code=200) _hot_cache={'t':0,'d':[]} def _get_hot_topics(): now=time.time() if _hot_cache['d'] and now-_hot_cache['t']<600:return _hot_cache['d'] freq={};display={} feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss'] for feed_url in feeds: try: r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=6);r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml') for item in soup.find_all('item')[:12]: title=_clean(item.find('title').get_text() if item.find('title') else '') if not title:continue title=re.sub(r'\s*[-|].*$','',title);words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP] if len(words)<2:continue for n in(3,4,2): for i in range(max(0,len(words)-n+1)): phrase=' '.join(words[i:i+n]) if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase except:continue ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True);topics=[];seen=set() for key,count in ranked: is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen) if is_dup:continue seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',display[key].title()),'topic':display[key],'count':count}) if len(topics)>=20:break for kw in['World Cup 2026','Kinh tế Việt Nam','Bóng đá châu Âu','Công nghệ AI','Giá vàng','Thời tiết']: if len(topics)>=24:break if not any(kw.lower() in s for s in seen):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw,'count':0}) _hot_cache.update({'t':now,'d':topics[:24]});return topics[:24] @app.get('/api/hot_topics') def api_hot_topics(): resp = JSONResponse({'topics':_get_hot_topics()}) resp.headers["Cache-Control"] = "public, max-age=120" return resp @app.get('/') async def serve_index(): p=os.path.join(STATIC_DIR,'index_v2.html') if os.path.exists(p):return FileResponse(p,media_type='text/html') return HTMLResponse('

VNEWS

') @app.get('/api/hashtag/sources') def _ht(topic:str=Query(...),page:int=Query(default=0)): items=_search_all(topic,36);per_page=8;start=page*per_page;end=start+per_page return JSONResponse({'sources':items[start:end],'topic':topic,'page':page,'has_more':end {_clean(safe_title)} ''' for s in slides: img_src = s.get('image', '') if img_src and ('cdnphoto.dantri' in img_src or 'refooty' in img_src or 'vnexpress' in img_src or 'vcdn' in img_src): img_tag = f'' else: img_tag = f'' if img_src else '' h += f'
Slide {s.get("index",1)}/{len(slides)}
{img_tag}

{_clean(s.get("text",""))}

' h += '' return HTMLResponse(h) def _render_video_page(post, safe_title, safe_img, safe_url): video_url = post.get('video', '') # Use text for description if available description = _clean((post.get('text') or '')[:200]) or "Tin tức tóm tắt, AI rewrite, World Cup 2026" # Build canonical URL preserving original query format if url was provided if safe_url and safe_url != '/': canonical_url = f"{SPACE}/s?url={quote(safe_url)}&title={quote(safe_title[:100])}" else: canonical_url = f"{SPACE}/s?post_id={post.get('id') or ''}" h = f''' {_clean(safe_title)}
{_clean(safe_title)}
''' return HTMLResponse(h) @app.get('/s/{slug}') async def _sh_slug(slug: str, request: Request, url: str = '', title: str = '', img: str = ''): """SEO-friendly share endpoint with slug in URL path. Shows slide content when slug matches a wall post ID, otherwise redirects. """ safe_title = _clean(title) if title else 'VNEWS - Tin tức' safe_img = _clean(img) if img else '' safe_url = _clean(url) if url else '/' # Try to find post by slug (post ID) post = None try: if slug and len(slug) > 5: # Likely a post ID posts = _load_wall_posts() for p in posts: if p.get('id') == slug: post = p safe_title = p.get('title', safe_title) or safe_title safe_img = p.get('img', safe_img) or safe_img safe_url = p.get('url', safe_url) or safe_url break except: pass if post and post.get('slides'): return _render_slides_page(post, safe_title, safe_img, safe_url) if post and post.get('video'): return _render_video_page(post, safe_title, safe_img, safe_url) # Otherwise redirect return HTMLResponse(f''' {_clean(safe_title)} ''') @app.get('/s') async def _sh(url:str='',title:str='',img:str='',post_id:str=''): safe_title = _clean(title) if title else 'VNEWS - Tin tức' safe_img = _clean(img) if img else '' safe_url = _clean(url) if url else '/' # Try to find wall post by post_id or URL (prioritize posts with slides/video) post = None try: posts = _load_wall_posts() if post_id: for p in posts: if p.get('id') == post_id: post = p safe_title = p.get('title', safe_title) or safe_title safe_img = p.get('img', safe_img) or safe_img safe_url = p.get('url', safe_url) or safe_url break elif url: # Find matching URL - prioritize posts with slides or video for p in posts: if p.get('url') == url and p.get('slides'): post = p safe_title = p.get('title', safe_title) or safe_title safe_img = p.get('img', safe_img) or safe_img safe_url = p.get('url', safe_url) or safe_url break if not post: # Fallback: find any matching URL for p in posts: if p.get('url') == url: post = p safe_title = p.get('title', safe_title) or safe_title safe_img = p.get('img', safe_img) or safe_img safe_url = p.get('url', safe_url) or safe_url break except: pass if post and post.get('slides'): return _render_slides_page(post, safe_title, safe_img, safe_url) if post and post.get('video'): return _render_video_page(post, safe_title, safe_img, safe_url) # Fallback: redirect to original URL return HTMLResponse(f''' {safe_title} ''') from wc2026_scraper import scrape_summary,scrape_fixtures,scrape_standings,scrape_stats,scrape_wc_news,scrape_road_to_wc,get_wc2026_all,scrape_history,scrape_h2h,scrape_lineups,scrape_match_detail _xlb_cache = {} _xlb_lock = threading.Lock() def _xlb_scrape(path): url = f"https://xemlaibongda.top/{path}" r = req.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}, timeout=15, allow_redirects=True) if r.status_code != 200: return [] soup = BeautifulSoup(r.text, 'lxml') vids = [] seen = set() for a in soup.select('a[href*="/video/"]'): href = a.get('href', '') if not href or href in seen: continue seen.add(href) if not href.startswith('http'): href = 'https://xemlaibongda.top' + href img = a.select_one('img') p = a.parent for _ in range(4): if img: break if p: img = p.select_one('img') p = p.parent img_src = '' if img: img_src = img.get('data-src','') or img.get('src','') or img.get('data-lazy','') or img.get('data-original','') if img_src.startswith('//'): img_src = 'https:' + img_src elif img_src.startswith('/'): img_src = 'https://xemlaibongda.top' + img_src title = '' for sel in ['.title', 'h3', 'h2', '.name', '.post-title', '.entry-title', '.video-title']: t = a.select_one(sel) if t: title = _clean(t.get_text()) break if not title: title = _clean(a.get('title','')) if not title: img_alt = a.select_one('img') if img_alt: title = _clean(img_alt.get('alt','')) if not title: parent = a.parent if parent: pt = _clean(parent.get_text(' ',strip=True)) if 5 < len(pt) < 120: title = pt if not title or len(title) < 3: continue vids.append({"link": href, "img": img_src, "title": title}) if len(vids) >= 30: break return vids @app.get('/api/proxy/xlb') def proxy_xlb(path: str = Query(default="")): now = time.time() cache_key = f"xlb:{path}" with _xlb_lock: cached = _xlb_cache.get(cache_key) if cached and now - cached['t'] < 120: return JSONResponse(cached['d']) try: vids = _xlb_scrape(path) result = {"videos": vids, "count": len(vids)} with _xlb_lock: _xlb_cache[cache_key] = {'t': now, 'd': result} return JSONResponse(result) except Exception as e: return JSONResponse({"videos": [], "count": 0, "error": str(e)}, status_code=500) @app.get('/api/wc2026') def _w():return JSONResponse(get_wc2026_all()) @app.get('/api/wc2026/fixtures') def _wf():return JSONResponse(scrape_fixtures()) @app.get('/api/wc2026/standings') def _ws():return JSONResponse(scrape_standings()) @app.get('/api/wc2026/stats') def _wst():return JSONResponse(scrape_stats()) @app.get('/api/wc2026/history') def _whi():return JSONResponse(scrape_history()) @app.get('/api/wc2026/news') def _wn():return JSONResponse(scrape_wc_news()) @app.get('/api/wc2026/road') def _wr():return JSONResponse(scrape_road_to_wc()) @app.get('/api/wc2026/h2h/{eid}') def _wh2(eid:int):return JSONResponse(scrape_h2h(eid)) @app.get('/api/wc2026/lineups/{eid}') def _wl(eid:int):return JSONResponse(scrape_lineups(eid)) @app.get('/api/wc2026/match/{eid}') def _wm(eid:int):return JSONResponse(scrape_match_detail(eid)) DATA_DIR='/data' if os.path.isdir('/data') else os.path.join(os.path.dirname(os.path.abspath(__file__)),'data') os.makedirs(DATA_DIR,exist_ok=True) IF=os.path.join(DATA_DIR,'interactions_v2.json') CF=os.path.join(DATA_DIR,'comments_v2.json') WALL_FILE=os.path.join(DATA_DIR,'wall_posts.json') WALL_VIDEO_DIR=os.path.join(DATA_DIR,'wall_videos') os.makedirs(WALL_VIDEO_DIR,exist_ok=True) _il=threading.Lock();_cl=threading.Lock();_wl_lock=threading.Lock() def _lj(p): try: if os.path.exists(p):return json.load(open(p,'r',encoding='utf-8')) except:pass return{} def _sj(p,d): try:open(p+'.tmp','w',encoding='utf-8').write(json.dumps(d,ensure_ascii=False));os.replace(p+'.tmp',p) except:pass @app.post('/api/v2/interact') async def _int(request:Request): b=await request.json();v=str(b.get('id','')).strip();t=str(b.get('type','')).strip() if not v or t not in('view','like'):return JSONResponse({'error':'x'},status_code=400) with _il:db=_lj(IF);db.setdefault(v,{'views':0,'likes':0,'comments':0});db[v][t+'s']+=1;_sj(IF,db);return JSONResponse(db[v]) @app.get('/api/v2/interactions') def _gi(id:str=Query(...)): with _il:return JSONResponse(_lj(IF).get(id.strip(),{'views':0,'likes':0,'comments':0})) @app.get('/api/v2/comments') def _gc(id:str=Query(...)): with _cl:return JSONResponse({'comments':_lj(CF).get(id.strip(),[])}) @app.post('/api/v2/comment') async def _pc(request:Request): b=await request.json();v=str(b.get('id','')).strip();tx=str(b.get('text','')).strip()[:500] if not v or not tx:return JSONResponse({'error':'x'},status_code=400) c={'text':tx,'time':time.strftime('%H:%M %d/%m',time.localtime()),'ts':int(time.time())} with _cl:db=_lj(CF);db.setdefault(v,[]);db[v].append(c);db[v]=db[v][-200:];_sj(CF,db);cms=db[v] with _il:idb=_lj(IF);idb.setdefault(v,{'views':0,'likes':0,'comments':0});idb[v]['comments']=len(cms);_sj(IF,idb) return JSONResponse({'comments':cms}) def _load_wall_posts(): with _wl_lock: return _lj(WALL_FILE) def _save_wall_posts(posts): with _wl_lock: _sj(WALL_FILE, posts) @app.get('/api/wall') def api_wall(): posts = _load_wall_posts() if not posts: return JSONResponse({"posts": []}) return JSONResponse({"posts": posts}) @app.post('/api/wall') async def api_wall_post(request: Request): content_type = request.headers.get('content-type', '') if 'multipart/form-data' in content_type: try: form = await request.form() except Exception as e: return JSONResponse({"error": f"Form parse error: {str(e)}"}, status_code=400) title = form.get('title', 'Video mới') or 'Video mới' text = form.get('text', '') or '' source = form.get('source', 'vtv_recorder') or 'vtv_recorder' video_file = form.get('video') post_id = str(uuid.uuid4())[:12] video_url = None if video_file and hasattr(video_file, 'filename') and video_file.filename: fname = video_file.filename.lower() if fname.endswith('.mp4'): ext = '.mp4' elif fname.endswith('.webm'): ext = '.webm' else: ext = '.webm' video_filename = f"wall_{post_id}{ext}" video_path = os.path.join(WALL_VIDEO_DIR, video_filename) try: content = await video_file.read() if not content: return JSONResponse({"error": "Empty video file"}, status_code=400) with open(video_path, 'wb') as f: f.write(content) file_size_mb = len(content) / 1024 / 1024 if file_size_mb > 50: os.remove(video_path) return JSONResponse({"error": f"Video quá lớn ({file_size_mb:.1f}MB). Tối đa 50MB."}, status_code=400) video_url = f"/api/wall/video/{video_filename}" except Exception as e: return JSONResponse({"error": f"Lỗi lưu video: {str(e)}"}, status_code=500) post = { "id": post_id, "title": title[:200], "text": text[:2000], "source": source, "video": video_url, "img": None, "images": [], "created": int(time.time()), "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()), } posts = _load_wall_posts() if not isinstance(posts, list): posts = [] posts.insert(0, post) posts = posts[:200] _save_wall_posts(posts) return JSONResponse({"post": post, "ok": True}) try: body = await request.json() except: body = {} title = body.get('title', 'Bài mới') or 'Bài mới' text = body.get('text', '') or '' img = body.get('img', None) source = body.get('source', 'user') or 'user' post_id = str(uuid.uuid4())[:12] post = { "id": post_id, "title": title[:200], "text": text[:2000], "source": source, "video": None, "img": img, "images": [], "created": int(time.time()), "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()), } posts = _load_wall_posts() if not isinstance(posts, list): posts = [] posts.insert(0, post) posts = posts[:200] _save_wall_posts(posts) return JSONResponse({"post": post, "ok": True}) @app.get('/api/wall/video/{filename}') def api_wall_video(filename: str): if '..' in filename or '/' in filename: return Response(status_code=403) video_path = os.path.join(WALL_VIDEO_DIR, filename) if not os.path.exists(video_path): return Response(status_code=404) ext = os.path.splitext(filename)[1].lower() media_type = 'video/mp4' if ext == '.mp4' else 'video/webm' return FileResponse(video_path, media_type=media_type) @app.delete('/api/wall/{post_id}') def api_wall_delete(post_id: str): posts = _load_wall_posts() if not isinstance(posts, list): return JSONResponse({"error": "No posts"}, status_code=404) for i, p in enumerate(posts): if p.get('id') == post_id: if p.get('video'): video_name = p['video'].split('/')[-1] video_path = os.path.join(WALL_VIDEO_DIR, video_name) if os.path.exists(video_path): os.remove(video_path) posts.pop(i) _save_wall_posts(posts) return JSONResponse({"ok": True}) return JSONResponse({"error": "Post not found"}, status_code=404) # ===== LANGUAGE & EMOTION DETECTION ===== import random as _random2 from urllib.parse import quote as _quote2 _UA_RW = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', 'Accept-Language': 'vi-VN,vi;q=0.9'} # Unique character markers for language detection _UNIQUE_CHARS = { 'vietnamese': set('đăâêôơưàảãạáằẳẵặắầẩẫậấèẻẽẹéềễểệếìỉĩịíòỏõọóồổỗộốờởỡợớùủũụúừửữựứỳỷỹỵý'), 'spanish': set('ñáéíóúü¿¡'), 'portuguese': set('ãõçáéíóúâêôà'), } _STOPWORDS = { 'english': {'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with', 'to', 'for', 'of', 'not', 'no', 'can', 'had', 'have', 'has', 'was', 'were', 'are', 'be', 'been', 'this', 'that', 'it', 'he', 'she', 'they', 'his', 'her', 'my', 'your', 'our', 'we', 'you', 'i'}, 'vietnamese': {'là', 'của', 'và', 'có', 'được', 'cho', 'không', 'với', 'này', 'đó', 'từ', 'trong', 'đã', 'sẽ', 'một', 'các', 'những', 'về', 'tại', 'người', 'năm', 'đến', 'ra', 'lại', 'như', 'khi', 'để', 'rất', 'cũng', 'mà', 'nếu', 'sau', 'trên', 'theo', 'vì', 'do', 'nên', 'thì', 'mình', 'tôi', 'bạn', 'anh', 'chị', 'em'}, 'portuguese': {'de', 'um', 'que', 'e', 'do', 'da', 'em', 'para', 'com', 'não', 'uma', 'os', 'no', 'se', 'na', 'por', 'mais', 'as', 'dos', 'como', 'mas', 'ao', 'ele', 'das', 'tem', 'seu', 'sua', 'ou', 'quando', 'muito', 'nos', 'já', 'eu', 'também', 'só', 'pelo', 'pela', 'até', 'isso', 'ela', 'entre', 'depois', 'sem', 'mesmo', 'aos', 'são', 'está', 'ter', 'ser', 'foi', 'era', 'há', 'estão', 'você', 'nós', 'eles', 'elas'}, 'spanish': {'de', 'que', 'el', 'en', 'y', 'a', 'los', 'del', 'se', 'las', 'por', 'un', 'para', 'con', 'no', 'una', 'su', 'al', 'es', 'lo', 'como', 'más', 'pero', 'sus', 'le', 'ya', 'o', 'fue', 'este', 'ha', 'si', 'porque', 'esta', 'son', 'entre', 'está', 'cuando', 'muy', 'sin', 'sobre', 'ser', 'también', 'me', 'hasta', 'hay', 'donde', 'han', 'quien', 'están', 'desde', 'todo', 'nos', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'fueron', 'ese', 'eso', 'ante', 'ellos', 'yo', 'tú', 'él', 'ella', 'nosotros', 'usted', 'ustedes'}, } def detect_language(text): """Detect language from text content using stopword + character analysis.""" if not text: return 'vietnamese' text_lower = text.lower() text_chars = set(text_lower) # Strong signal: Vietnamese unique characters vn_chars = len(text_chars & _UNIQUE_CHARS['vietnamese']) if vn_chars >= 2: return 'vietnamese' # Spanish unique chars (ñ, ¿, ¡) es_chars = len(text_chars & _UNIQUE_CHARS['spanish']) pt_chars = len(text_chars & _UNIQUE_CHARS['portuguese']) # Stopword scoring words = set(re.findall(r'\b\w+\b', text_lower)) scores = {} for lang, stops in _STOPWORDS.items(): scores[lang] = len(words & stops) / max(len(stops), 1) # Disambiguate Portuguese vs Spanish pt_markers = {'não', 'pelo', 'pela', 'isso', 'há', 'estão', 'num', 'numa', 'tenho', 'posso', 'você', 'nós', 'eles', 'elas', 'também', 'muito', 'já', 'só', 'até', 'entre', 'depois', 'sem', 'mesmo', 'aos', 'serão'} es_markers = {'pero', 'está', 'están', 'porque', 'también', 'hasta', 'donde', 'quien', 'fue', 'son', 'fueron', 'ese', 'eso', 'ante', 'ellos', 'ella', 'nosotros', 'usted', 'ustedes', 'tú', 'él', 'desde', 'todo', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'fueron'} pt_overlap = len(words & pt_markers) es_overlap = len(words & es_markers) if scores.get('portuguese', 0) > 0 and pt_overlap > es_overlap: return 'portuguese' if scores.get('spanish', 0) > 0 and es_overlap > pt_overlap: return 'spanish' if scores.get('english', 0) > 0.15: return 'english' best = max(scores, key=scores.get) return best if scores[best] > 0.05 else 'vietnamese' # Emotion keyword-based detection _EMOTION_KEYWORDS = { 'happy': { 'en': ['happy', 'joy', 'wonderful', 'great', 'amazing', 'fantastic', 'love', 'excellent', 'beautiful', 'glad', 'delighted', 'pleased', 'cheerful', 'celebrate', 'victory', 'win', 'success'], 'pt': ['feliz', 'alegria', 'maravilhoso', 'ótimo', 'incrível', 'fantástico', 'amor', 'excelente', 'lindo', 'contente', 'encantado', 'vitória', 'sucesso'], 'es': ['feliz', 'alegria', 'maravilloso', 'genial', 'increíble', 'fantástico', 'amor', 'excelente', 'hermoso', 'contento', 'encantado', 'victoria', 'éxito'], 'vi': ['vui', 'hạnh phúc', 'tuyệt vời', 'tuyệt', 'ý nghĩa', 'đẹp', 'thích', 'yêu', 'vui vẻ', 'hân hoan', 'phấn khích', 'chiến thắng', 'thành công'], }, 'sad': { 'en': ['sad', 'unhappy', 'terrible', 'awful', 'horrible', 'miserable', 'depressed', 'grief', 'sorrow', 'tragic', 'unfortunate', 'painful', 'death', 'die', 'kill'], 'pt': ['triste', 'infeliz', 'terrível', 'horrível', 'miserável', 'deprimido', 'dor', 'trágico', 'infelizmente', 'penoso', 'morte', 'morrer'], 'es': ['triste', 'infeliz', 'terrible', 'horrible', 'miserable', 'deprimido', 'dolor', 'trágico', 'desafortunado', 'penoso', 'muerte', 'morir'], 'vi': ['buồn', 'không vui', 'tồi tệ', 'kinh khủng', 'đau khổ', 'đau buồn', 'bi thương', 'khốn nạn', 'đau đớn', 'thảm họa', 'chết', 'mất'], }, 'excited': { 'en': ['excited', 'thrilling', 'amazing', 'wow', 'incredible', 'unbelievable', 'awesome', 'exhilarating', 'electrifying', 'breathtaking', 'breakthrough', 'record'], 'pt': ['animado', 'emocionante', 'incrível', 'impressionante', 'sensacional', 'eletrizante', 'empolgante', 'recorde'], 'es': ['emocionante', 'increíble', 'impresionante', 'sensacional', 'electrizante', 'emocionado', 'entusiasmado', 'récord'], 'vi': ['hào hứng', 'phấn khích', 'thú vị', 'tuyệt cú mèo', 'đỉnh cao', 'ngoạn mục', 'sục sôi', 'kỷ lục', 'đột phá'], }, 'humorous': { 'en': ['funny', 'hilarious', 'joke', 'laugh', 'comedy', 'humor', 'amusing', 'witty', 'sarcastic', 'ironic', 'ridiculous', 'absurd', 'lol', 'haha'], 'pt': ['engraçado', 'hilário', 'piada', 'rir', 'comédia', 'humor', 'divertido', 'irônico', 'ridículo', 'absurdo', 'kkk'], 'es': ['gracioso', 'hilarante', 'broma', 'risa', 'comedia', 'humor', 'divertido', 'irónico', 'ridículo', 'absurdo', 'jaja'], 'vi': ['hài hước', 'buồn cười', 'đùa', 'cười', 'hài', 'vui nhộn', 'hóm hỉnh', 'mỉa mai', 'lố bịch', 'vô lý', 'haha'], }, 'serious': { 'en': ['serious', 'critical', 'important', 'urgent', 'severe', 'grave', 'significant', 'crucial', 'vital', 'essential', 'alarming', 'concerning', 'crisis', 'war', 'conflict'], 'pt': ['sério', 'crítico', 'importante', 'urgente', 'grave', 'significativo', 'crucial', 'vital', 'essencial', 'preocupante', 'crise', 'guerra', 'conflito'], 'es': ['serio', 'crítico', 'importante', 'urgente', 'grave', 'significativo', 'crucial', 'vital', 'esencial', 'preocupante', 'crisis', 'guerra', 'conflicto'], 'vi': ['nghiêm trọng', 'quan trọng', 'khẩn cấp', 'nghiêm túc', 'đáng kể', 'thiết yếu', 'cần thiết', 'báo động', 'lo ngại', 'khủng hoảng', 'chiến tranh', 'xung đột'], }, } def detect_emotion(text, language='vietnamese'): """Detect emotion from text using keyword matching.""" if not text: return 'neutral' text_lower = text.lower() scores = {} for emotion, lang_keywords in _EMOTION_KEYWORDS.items(): keywords = lang_keywords.get(language, lang_keywords.get('en', [])) score = sum(1 for kw in keywords if kw in text_lower) scores[emotion] = score if max(scores.values()) == 0: return 'neutral' return max(scores, key=scores.get) def detect_language_and_emotion(title, text): """Detect both language and emotion from article content.""" combined = f"{title} {text}" lang = detect_language(combined) emotion = detect_emotion(combined, lang) return lang, emotion # Voice selection based on language and emotion (using MultilingualNeural voices) VOICE_BY_LANG_EMOTION = { 'vietnamese': { 'happy': ('vi-VN-HoaiMyNeural', 'vui'), 'sad': ('vi-VN-NamMinhNeural', 'buồn'), 'excited': ('vi-VN-HoaiMyNeural', 'hào hứng'), 'humorous': ('vi-VN-HoaiMyNeural', 'vui'), 'serious': ('vi-VN-NamMinhNeural', 'nghiêm túc'), 'neutral': ('vi-VN-HoaiMyNeural', 'trung_tinh'), }, 'portuguese': { 'happy': ('pt-BR-ThalitaMultilingualNeural', 'feliz'), 'sad': ('pt-BR-ThalitaMultilingualNeural', 'triste'), 'excited': ('pt-BR-ThalitaMultilingualNeural', 'animado'), 'humorous': ('pt-BR-ThalitaMultilingualNeural', 'engraçado'), 'serious': ('pt-BR-ThalitaMultilingualNeural', 'sério'), 'neutral': ('pt-BR-ThalitaMultilingualNeural', 'neutro'), }, 'english': { 'happy': ('en-US-AndrewMultilingualNeural', 'happy'), 'sad': ('en-AU-WilliamMultilingualNeural', 'sad'), 'excited': ('en-US-AndrewMultilingualNeural', 'excited'), 'humorous': ('en-US-AndrewMultilingualNeural', 'funny'), 'serious': ('en-AU-WilliamMultilingualNeural', 'serious'), 'neutral': ('en-US-AndrewMultilingualNeural', 'neutral'), }, 'french': { 'happy': ('fr-FR-VivienneMultilingualNeural', 'heureux'), 'sad': ('fr-FR-RemyMultilingualNeural', 'triste'), 'excited': ('fr-FR-VivienneMultilingualNeural', 'excité'), 'humorous': ('fr-FR-VivienneMultilingualNeural', 'drôle'), 'serious': ('fr-FR-RemyMultilingualNeural', 'sérieux'), 'neutral': ('fr-FR-VivienneMultilingualNeural', 'neutre'), }, 'german': { 'happy': ('de-DE-SeraphinaMultilingualNeural', 'glücklich'), 'sad': ('de-DE-FlorianMultilingualNeural', 'traurig'), 'excited': ('de-DE-SeraphinaMultilingualNeural', 'aufgeregt'), 'humorous': ('de-DE-SeraphinaMultilingualNeural', 'lustig'), 'serious': ('de-DE-FlorianMultilingualNeural', 'ernst'), 'neutral': ('de-DE-SeraphinaMultilingualNeural', 'neutral'), }, 'korean': { 'happy': ('ko-KR-HyunsuMultilingualNeural', '행복'), 'sad': ('ko-KR-HyunsuMultilingualNeural', '슬픔'), 'excited': ('ko-KR-HyunsuMultilingualNeural', '흥분'), 'humorous': ('ko-KR-HyunsuMultilingualNeural', '유쾌'), 'serious': ('ko-KR-HyunsuMultilingualNeural', '진지'), 'neutral': ('ko-KR-HyunsuMultilingualNeural', '중립'), }, 'italian': { 'happy': ('it-IT-GiuseppeMultilingualNeural', 'felice'), 'sad': ('it-IT-GiuseppeMultilingualNeural', 'triste'), 'excited': ('it-IT-GiuseppeMultilingualNeural', 'emozionato'), 'humorous': ('it-IT-GiuseppeMultilingualNeural', 'divertente'), 'serious': ('it-IT-GiuseppeMultilingualNeural', 'serio'), 'neutral': ('it-IT-GiuseppeMultilingualNeural', 'neutro'), }, } # All valid voice IDs (new MultilingualNeural format) VALID_VOICES = { 'vi-VN-HoaiMyNeural', 'vi-VN-NamMinhNeural', 'en-US-AndrewMultilingualNeural', 'en-AU-WilliamMultilingualNeural', 'pt-BR-ThalitaMultilingualNeural', 'fr-FR-VivienneMultilingualNeural', 'fr-FR-RemyMultilingualNeural', 'de-DE-SeraphinaMultilingualNeural', 'de-DE-FlorianMultilingualNeural', 'ko-KR-HyunsuMultilingualNeural', 'it-IT-GiuseppeMultilingualNeural', } def get_voice_for_content(title, text, preferred_voice=None): """Get appropriate voice based on content language and emotion.""" # Accept the new MultilingualNeural voices directly if preferred_voice and preferred_voice in VALID_VOICES: return preferred_voice # Also accept old shorthand voice IDs and map them to new format old_voice_map = { 'hoaimy': 'vi-VN-HoaiMyNeural', 'namminh': 'vi-VN-NamMinhNeural', 'andrew': 'en-US-AndrewMultilingualNeural', 'jenny': 'en-US-AndrewMultilingualNeural', 'thalita': 'pt-BR-ThalitaMultilingualNeural', 'pt_thalita': 'pt-BR-ThalitaMultilingualNeural', 'pt_francisco': 'pt-BR-ThalitaMultilingualNeural', 'ela': 'en-US-AndrewMultilingualNeural', 'es_carlos': 'en-US-AndrewMultilingualNeural', 'denise': 'fr-FR-VivienneMultilingualNeural', 'katja': 'de-DE-SeraphinaMultilingualNeural', 'nanami': 'en-US-AndrewMultilingualNeural', 'sunhee': 'ko-KR-HyunsuMultilingualNeural', 'xiaochen': 'en-US-AndrewMultilingualNeural', } if preferred_voice and preferred_voice in old_voice_map: return old_voice_map[preferred_voice] lang, emotion = detect_language_and_emotion(title, text) lang_map = VOICE_BY_LANG_EMOTION.get(lang, VOICE_BY_LANG_EMOTION['vietnamese']) voice, _ = lang_map.get(emotion, lang_map['neutral']) return voice def _is_relevant_image(img_url, title, text): """Check if an image is relevant to the article content.""" if not img_url: return False skip_patterns = ['pixel', 'analytics', 'tracking', '1x1.gif', 'spacer.gif', 'logo', 'icon', 'avatar', 'emoji', 'smiley', 'sprite', 'advertisement', 'ad-banner', 'sponsored', 'banner-ads'] img_lower = img_url.lower() for p in skip_patterns: if p in img_lower: return False if not any(img_lower.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.webp', '.gif']): return False return True def _filter_relevant_images(images, title, text, max_images=8): """Filter and rank images by relevance to article content.""" if not images: return [] seen = set() relevant = [] for img in images: if img in seen: continue seen.add(img) if _is_relevant_image(img, title, text): relevant.append(img) return relevant[:max_images] def _scrape_article_for_rewrite(url): """Scrape article: extract title, paragraphs, RELEVANT images, OG image.""" try: r = req.get(url, headers=_UA_RW, timeout=15, allow_redirects=True) r.encoding = 'utf-8' soup = BeautifulSoup(r.text, 'lxml') for tag in soup.find_all(['script', 'style', 'nav', 'footer', 'aside', 'form']): tag.decompose() h1 = soup.find('h1') ogt = soup.find('meta', property='og:title') title = (h1.get_text(strip=True) if h1 else '') or (ogt.get('content', '') if ogt else '') ogi = soup.find('meta', property='og:image') og_img = ogi.get('content', '') if ogi else '' if og_img and og_img.startswith('//'): og_img = 'https:' + og_img block = None for sel in ['article', '.singular-content', '.detail-content', '.fck_detail', '.content-detail', '.knc-content', 'main', '.cms-body', '.article__body']: el = soup.select_one(sel) if el and len(el.find_all('p')) >= 2: block = el break if not block: block = soup.body or soup paragraphs = [] all_images = [] seen_imgs = set() if og_img and og_img not in seen_imgs: all_images.append(og_img) seen_imgs.add(og_img) for el in block.find_all(['p', 'h2', 'h3', 'figure', 'img'], recursive=True): if el.name == 'p': t = _clean(el.get_text(strip=True)) if t and len(t) > 40: paragraphs.append(t) elif el.name in ('figure', 'img'): im = el if el.name == 'img' else el.find('img') if im: src = im.get('data-src') or im.get('src') or im.get('data-original') or '' if src and 'base64' not in src: if src.startswith('//'): src = 'https:' + src if src not in seen_imgs: all_images.append(src) seen_imgs.add(src) # Filter to relevant images only relevant_images = _filter_relevant_images(all_images, title, ' '.join(paragraphs[:5])) return {'title': _clean(title), 'paragraphs': paragraphs, 'images': relevant_images, 'og_img': og_img} except Exception: return None def _extract_key_points_rw(paragraphs, max_points=5): r"""Extract key points from paragraphs - extracts ALL sentences, not just first one. Fixes: Original regex `^(.+?[.!?])\s` only captured first sentence per paragraph. Now splits on all sentence boundaries and takes valid sentences until max_points. """ points = [] for p in paragraphs: if len(points) >= max_points: break p = _clean(p) if not p: continue # Split paragraph into sentences using Vietnamese + English punctuation sentences = re.split(r'(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])', p) sentences = [s.strip() for s in sentences if s.strip()] for sentence in sentences: if len(points) >= max_points: break # Clean sentence - remove extra whitespace sentence = _clean(sentence) if len(sentence) < 30: continue # Check for duplicates if any(sentence[:60] in existing for existing in points): continue # Ensure sentence ends with punctuation if not sentence.endswith(('.', '!', '?')): sentence = sentence + '.' points.append(sentence) # If no valid sentences found, take chunks from raw text if not points: raw = '\n'.join(paragraphs) for i in range(0, min(len(raw), max_points * 300), 280): chunk = _clean(raw[i:i+280]) if len(chunk) >= 30 and chunk not in points: points.append(chunk + ('.' if not chunk.endswith('.') else '')) if len(points) >= max_points: break return points @app.post("/api/rewrite_slide") async def api_rewrite_slide(request: Request): """Fast rewrite as SLIDES - no AI needed, instant response.""" body = await request.json() url = _clean(body.get("url", "")) context = body.get("context", "") preferred_voice = body.get("voice", "") # Accept custom voice selection if not url and not context: return JSONResponse({"error": "Cần URL hoặc nội dung"}, status_code=400) data = None if url and url.startswith("http"): data = _scrape_article_for_rewrite(url) if not data and context: paragraphs = [_clean(p) for p in context.split('\n') if len(_clean(p)) > 40] data = {'title': paragraphs[0][:80] if paragraphs else 'Bài viết', 'paragraphs': paragraphs, 'images': [], 'og_img': ''} if not data or not data.get('paragraphs'): return JSONResponse({"error": "Không đọc được bài viết"}, status_code=422) points = _extract_key_points_rw(data['paragraphs'], max_points=12) if not points: return JSONResponse({"error": "Không tìm được ý chính"}, status_code=422) images = data.get('images', []) slides = [] for i, point in enumerate(points): img = images[i] if i < len(images) else (images[-1] if images else '') if img and 'cdnphoto.dantri' in img: img = '/api/proxy/img?url=' + _quote2(img, safe='') slides.append({'text': point, 'image': img, 'index': i + 1}) summary_text = '\n\n'.join([f"• {s['text']}" for s in slides]) # Auto-detect language and emotion lang, emotion = detect_language_and_emotion(data['title'], summary_text) # Use preferred voice if provided, otherwise auto-detect voice = preferred_voice if preferred_voice else get_voice_for_content(data['title'], summary_text) post = { "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)), "title": data['title'], "text": summary_text, "img": images[0] if images else '', "url": url, "kind": "slide_summary", "slides": slides, "images": images[:10], "video": "", "voice": voice, "emotion": emotion, "language": lang, "ts": int(time.time()) } posts = _load_wall_posts() posts.insert(0, post) _save_wall_posts(posts) return JSONResponse({"post": post, "slides": slides}) @app.post("/api/rewrite_share") async def api_rewrite_share(request: Request): """Rewrite article and post to Tường AI with SLIDES + AI text.""" body = await request.json() url = _clean(body.get("url", "")) ctx = _clean(body.get("context", "")) preferred_voice = body.get("voice", "") # Accept custom voice selection if not url and not ctx: return JSONResponse({"error": "Cần URL hoặc nội dung"}, status_code=400) data = None if url and url.startswith("http"): data = _scrape_article_for_rewrite(url) if not data and ctx: paragraphs = [_clean(p) for p in ctx.split('\n') if len(_clean(p)) > 40] data = {'title': paragraphs[0][:80] if paragraphs else 'Bài viết', 'paragraphs': paragraphs, 'images': [], 'og_img': ''} if not data or not data.get('paragraphs'): return JSONResponse({"error": "Không đọc được bài viết"}, status_code=422) raw_text = '\n'.join(data['paragraphs']) if len(raw_text) < 50: raw_text = ctx[:14000] if len(raw_text) < 50: return JSONResponse({"error": "Bài viết quá ngắn"}, status_code=422) domain = '' try: from urllib.parse import urlparse domain = urlparse(url).netloc.replace('www.', '') except: pass # Generate AI summary text ai_text = None try: import ai_ext if hasattr(ai_ext, 'qwen_generate'): prompt = f'Tóm tắt đăng Tường AI:\nTiêu đề: {data["title"]}\n{raw_text[:14000]}\n\n4-6 ý chính. Cuối ghi nguồn.' ai_text = await ai_ext.qwen_generate(prompt, max_tokens=1000) except Exception: pass if not ai_text or len(ai_text) < 80: key_pts = _extract_key_points_rw(data['paragraphs'], max_points=12) if key_pts: ai_text = '\n\n'.join([f"• {p}" for p in key_pts]) else: ai_text = f"Tóm tắt: {data['title']}\n\n{raw_text[:1200]}\n\nNguồn: {domain}" # Build slides from key points (FIX: include slides in rewrite_share too!) points = _extract_key_points_rw(data['paragraphs'], max_points=12) images = data.get('images', []) slides = [] for i, point in enumerate(points): img = images[i] if i < len(images) else (images[-1] if images else '') if img and 'cdnphoto.dantri' in img: img = '/api/proxy/img?url=' + _quote2(img, safe='') slides.append({'text': point, 'image': img, 'index': i + 1}) # Auto-detect language and emotion lang, emotion = detect_language_and_emotion(data['title'], ai_text) # Use preferred voice if provided, otherwise auto-detect voice = preferred_voice if preferred_voice else get_voice_for_content(data['title'], ai_text) post = { "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)), "title": data['title'], "text": ai_text, "img": images[0] if images else '', "url": url, "kind": "rewrite", "slides": slides, "images": images[:10], "video": "", "voice": voice, "emotion": emotion, "language": lang, "ts": int(time.time()) } posts = _load_wall_posts() posts.insert(0, post) _save_wall_posts(posts) return JSONResponse({"post": post, "slides": slides}) @app.post("/api/url_wall") async def api_url_wall(request: Request): """Submit URL to add to Tường AI.""" body = await request.json() url = _clean(body.get("url", "")) if not url or not url.startswith('http'): return JSONResponse({"error": "URL không hợp lệ"}, status_code=400) # Reuse rewrite_share logic req._body = json.dumps({"url": url}).encode() return await api_rewrite_share(request) def _bg(): time.sleep(15) while True: try:get_wc2026_all() except:pass time.sleep(90) threading.Thread(target=_bg,daemon=True).start() # ===== AUTO SCHEDULER: rewrite AI + short at 7/13/19 VN time ===== _AUTO_SCHEDULE_TIMES = [(7, '07:00'), (13, '13:00'), (19, '19:00')] _AUTO_LOG = os.path.join(DATA_DIR, 'auto_rewrite_log.json') def _load_auto_log(): try: if os.path.exists(_AUTO_LOG): with open(_AUTO_LOG, 'r') as f: return json.load(f) except: pass return {} def _save_auto_log(log): try: tmp = _AUTO_LOG + '.tmp' with open(tmp, 'w') as f: json.dump(log, f) os.replace(tmp, _AUTO_LOG) except: pass async def _auto_fetch_short(post_id): """Try to auto-generate a short for a post.""" try: import httpx async with httpx.AsyncClient(timeout=180) as cl: r = await cl.post( f"http://localhost:7860/api/ai/short/{post_id}", json={"voice":"vi-VN-HoaiMyNeural","emotion":"neutral","speed":1.2}, headers={"Content-Type":"application/json"} ) if r.status_code < 300: sj = r.json() if sj.get('video'): posts = _load_wall_posts() for p in posts: if p.get('id') == post_id: p['video'] = sj['video'] break _save_wall_posts(posts) return True except: pass return False async def _auto_rewrite_one(topic, slot_label, used_urls=None, post_index=0): """Rewrite one topic: find articles, summarize, post to wall, trigger short. used_urls: shared set to avoid duplicate articles across topics. post_index: 0-based index to create multiple posts per topic (0,1,2 = up to 3 posts).""" from urllib.parse import quote as _q # Get MORE items to support 1-3 posts per topic items = _search_all(topic, limit=12) # Skip URLs already used by another topic if used_urls is not None: filtered = [it for it in items if it.get('url') not in used_urls] if filtered: items = filtered if not items or post_index >= len(items): return False # Get article at post_index (0,1,2 for multiple posts) item = items[post_index] # post_index allows multiple articles per topic url = item.get('url', '') title = item.get('title', topic) if url and used_urls is not None: used_urls.add(url) if not url.startswith('http'): return False data = _scrape_article_for_rewrite(url) if not data or not data.get('paragraphs'): return False raw_text = '\n'.join(data['paragraphs']) ai_text = None # Try AI generation try: import ai_ext prompt = f"Tóm tắt tin tức (tự động {slot_label}):\nTiêu đề: {data['title']}\n{raw_text[:10000]}\n\n4-6 ý chính dạng bullet. Cuối ghi nguồn." ai_text = await ai_ext.qwen_generate(prompt, max_tokens=1000) except: pass if not ai_text or len(ai_text) < 80: pts = data['paragraphs'][:6] ai_text = '\n\n'.join([f"• {p[:300]}" for p in pts]) via = item.get('via', '') or urlparse(url).netloc.replace('www.', '') ai_text += f"\n\nNguồn tham khảo: {via}" # Build slides images = data.get('images', []) pts = data['paragraphs'][:10] slides = [] for i, p in enumerate(pts[:8]): img = images[i] if i < len(images) else (images[-1] if images else data.get('og_img', '')) slides.append({'text': p[:300], 'image': img, 'index': i + 1}) post_id = str(int(time.time() * 1000)) + str(_random2.randint(100, 999)) post = { "id": post_id, "title": data.get('title', title)[:200], "text": ai_text, "img": images[0] if images else data.get('og_img', ''), "url": url, "kind": "auto_rewrite", "slides": slides, "images": images[:10], "video": "", "voice": "vi-VN-HoaiMyNeural", "emotion": "neutral", "language": "vietnamese", "ts": int(time.time()), "auto_scheduled": True, "slot": slot_label, } posts = _load_wall_posts() posts.insert(0, post) _save_wall_posts(posts) # Trigger short generation async threading.Thread(target=lambda: asyncio.run(_auto_fetch_short(post_id)), daemon=True).start() return True async def _do_scheduled_run(slot_label): """Main scheduled run: 1-3 posts from 3 different HOT topics (3-9 total), no duplicates.""" print(f"[auto] Starting scheduled rewrite for {slot_label}") # Get top hot topics, skip duplicates all_topics = _get_hot_topics() seen_topics = set() unique_topics = [] for t in all_topics: kw = t.get('topic', '').lower().strip() if kw and len(kw) > 5 and kw not in seen_topics: is_dup = False for s in seen_topics: # Check if one topic is substring of another if kw in s or s in kw: is_dup = True break if not is_dup: seen_topics.add(kw) unique_topics.append(t) if len(unique_topics) >= 3: break job_topics = [t['topic'] for t in unique_topics[:3] if t.get('topic')] if not job_topics: print(f"[auto] No hot topics found, skipping") return print(f"[auto] Running 3 topics: {job_topics}") # Track used URLs to avoid cross-topic duplicates _used_urls = set() results = [] # Process each topic, create 1-3 posts per topic for jt in job_topics: for post_idx in range(3): # Try up to 3 posts per topic try: ok = await asyncio.wait_for(_auto_rewrite_one(jt, slot_label, _used_urls, post_idx), timeout=120) if ok: results.append((jt, post_idx, True)) print(f"[auto] Created post {post_idx+1} for '{jt}'") else: # No more articles for this topic break except Exception as e: print(f"[auto] Error on '{jt}' post {post_idx}: {e}") results.append((jt, post_idx, False)) await asyncio.sleep(1) # Small delay between posts # Ensure at least 3 posts total (fallback if needed) successful_posts = sum(1 for _, _, ok in results if ok) print(f"[auto] Done {slot_label}: {successful_posts} posts created") # Log from datetime import datetime, timezone, timedelta VN_TZ_SCHED = timezone(timedelta(hours=7)) today_str = datetime.now(VN_TZ_SCHED).strftime('%Y-%m-%d') log = _load_auto_log() if today_str not in log: log[today_str] = {} log[today_str][slot_label] = { 'time': datetime.now(VN_TZ_SCHED).strftime('%H:%M:%S'), 'count': successful_posts, 'total': len(job_topics), } _save_auto_log(log) def _scheduler_loop(): """Check every 60s; trigger at 7:00, 13:00, 19:00 VN time. On startup, check for any missed slots today and run them immediately.""" time.sleep(35) from datetime import datetime, timezone, timedelta VN_TZ_SCHED = timezone(timedelta(hours=7)) _last_run_date = "" _last_run_slots = set() # On startup: check log for missed slots today try: start_now = datetime.now(VN_TZ_SCHED) today_str = start_now.strftime('%Y-%m-%d') current_hour = start_now.hour current_minute = start_now.minute log = _load_auto_log() today_log = log.get(today_str, {}) for h, label in _AUTO_SCHEDULE_TIMES: # Run if slot is past (either strictly earlier hour, or same hour but window has passed) should_run = False if h < current_hour: should_run = True elif h == current_hour and current_minute > 10: should_run = True if should_run and label not in today_log: print(f"[auto] Detected missed slot {label} (h={h} < now={current_hour}:{current_minute}), running catch-up now") _run_scheduled_sync(label) _last_run_slots.add(label) except Exception as e: print(f"[auto] Catch-up check error: {e}") while True: try: now = datetime.now(VN_TZ_SCHED) today = now.strftime('%Y-%m-%d') hour = now.hour minute = now.minute if today != _last_run_date: _last_run_date = today _last_run_slots = set() slot = None for h, label in _AUTO_SCHEDULE_TIMES: if hour == h and 0 <= minute < 5: slot = label break if slot and slot not in _last_run_slots: _last_run_slots.add(slot) _run_scheduled_sync(slot) except Exception as e: print(f"[auto] Loop error: {e}") time.sleep(60) threading.Thread(target=_scheduler_loop, daemon=True, name='auto-rewrite-scheduler').start() @app.get('/api/debug/auto_schedule') async def debug_auto_schedule(slot: str = '07:00'): """Manually trigger auto scheduler for debugging.""" try: # Check if we can access the data directory log = _load_auto_log() topics = _get_hot_topics()[:3] job_topics = [t['topic'] for t in topics if t.get('topic')] return JSONResponse({ "slot": slot, "log": log, "hot_topics": job_topics, "wall_posts_count": len(_load_wall_posts()), "data_dir_writable": os.access(DATA_DIR, os.W_OK) if os.path.isdir(DATA_DIR) else False, "data_dir_exists": os.path.isdir(DATA_DIR), }) except Exception as e: return JSONResponse({"error": str(e)}, status_code=500) def _run_scheduled_sync(slot): """Run _do_scheduled_run in a separate event loop (for background thread).""" loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: loop.run_until_complete(_do_scheduled_run(slot)) except Exception as e: print(f"[auto] Background run error: {e}") finally: loop.close() @app.get('/api/debug/trigger_auto') async def debug_trigger_auto(slot: str = '19:00'): """Trigger _do_scheduled_run in background thread (non-blocking).""" threading.Thread(target=_run_scheduled_sync, args=(slot,), daemon=True).start() return JSONResponse({"status": "started", "slot": slot}) # ===== SHORTS RSS PROXY ENDPOINT ===== @app.get("/api/shorts/rss") def shorts_rss(): """Get shorts from YouTube RSS feeds server-side""" import xml.etree.ElementTree as ET import html as html_lib2 import re as re2 YOUTUBE_CHANNELS = { "baodantri7941": "UC_x5TKhOgd6GhYvv5z4I3jg", "baosuckhoedoisongboyte": "UCBsY5fXTQLkF_JnH9kLkL4g", } shorts = [] seen = set() for handle, channel_id in YOUTUBE_CHANNELS.items(): try: rss_url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}" r = req.get(rss_url, headers=HEADERS, timeout=15) if r.status_code != 200: continue root = ET.fromstring(r.text) ns = { 'atom': 'http://www.w3.org/2005/Atom', 'yt': 'http://www.youtube.com/xml/schemas/2015', 'media': 'http://search.yahoo.com/mrss/' } for entry in root.findall('atom:entry', ns)[:30]: title_el = entry.find('atom:title', ns) title = html_lib2.unescape(title_el.text) if title_el is not None and title_el.text else '' link_el = entry.find('atom:link', ns) link = link_el.get('href', '') if link_el is not None else '' vid_el = entry.find('yt:videoId', ns) vid = vid_el.text if vid_el is not None else '' if not vid or vid in seen: continue # Check if it's a short is_short = '#shorts' in title.lower() or '#short' in title.lower() or '/shorts/' in link if not is_short: desc_el = entry.find('media:description', ns) if desc_el is not None and desc_el.text: if '#shorts' in desc_el.text.lower(): is_short = True if not is_short: continue seen.add(vid) # Get thumbnail thumb = f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg" media_group = entry.find('media:group', ns) if media_group is not None: thumb_el = media_group.find('media:thumbnail', ns) if thumb_el is not None: thumb = thumb_el.get('url', thumb) shorts.append({ 'id': vid, 'title': title.replace('#shorts', '').replace('#short', '').strip()[:120], 'img': thumb, 'link': f'https://www.youtube.com/shorts/{vid}', 'channel': handle, 'source': 'yt' }) if len(shorts) >= 40: break except Exception as e: print(f"RSS error for {handle}: {e}") continue return {"shorts": shorts, "count": len(shorts)} app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')