diff --git "a/app_v2_entry.py" "b/app_v2_entry.py" --- "a/app_v2_entry.py" +++ "b/app_v2_entry.py" @@ -1,4 +1,4 @@ -"""VNEWS v2 Entry Point - with fast bongda proxy + rewrite endpoints + multilingual TTS""" +"""VNEWS v2 Entry Point - with fast bongda proxy""" import sys, os from main import app, HEADERS, BONGDA_HEADERS, fetch_bongda_api, HL_LEAGUES @@ -7,16 +7,6 @@ try: except Exception as e: print(f"[WARN] ai_ext import failed: {e}") -try: - import ai_patch -except Exception as e: - print(f"[WARN] ai_patch import failed: {e}") - -try: - import ai_short_v2 # Short AI v2: TikTok music, uploaded audio/video/img, recreate from slides -except Exception as e: - print(f"[WARN] ai_short_v2 import failed: {e}") - from fastapi.responses import HTMLResponse, JSONResponse, FileResponse, Response from fastapi.staticfiles import StaticFiles from starlette.routing import Mount @@ -26,35 +16,16 @@ from bs4 import BeautifulSoup import re, html as html_lib, json, threading, time, uuid from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import quote -import asyncio HL_LEAGUES['friendly'] = {"path": "giai-khac/friendly", "name": "Giao hữu", "emoji": "🤝"} STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static') -SPACE = "https://bep40-vnews.hf.space" # SEO URL base for share links app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/' and hasattr(r,'methods') and 'GET' in getattr(r,'methods',set()))] app.routes[:]=[r for r in app.routes if not isinstance(r, Mount)] app.router.routes=[r for r in app.router.routes if not isinstance(r, Mount)] def _clean(s): return re.sub(r"\s+"," ",html_lib.unescape(str(s or""))).strip() -def _ensure_sentence_complete(text): - """Ensure text ends with complete sentence (ends with . ! or ?). Trim if cut mid-sentence.""" - text = _clean(text) - if not text: - return text - # Find last sentence ending - for end_char in ['.', '!', '?']: - last_pos = text.rfind(end_char) - if last_pos > len(text) * 0.5: # Keep if ending is in latter half - return text[:last_pos + 1].strip() - # If no ending found, try to find last complete sentence - sentences = re.split(r'(?<=[.!?])\s+', text) - complete = [s.strip() for s in sentences if s.strip() and len(s.strip()) > 20] - if complete[:-1]: # Return all but last incomplete - return ' '.join(complete[:-1]) - return text[:150] + '.' if len(text) > 150 else text - # Cache for match details (5 min TTL) _match_cache = {} @@ -231,7 +202,7 @@ _STOP=set('và của các những một được trong với cho tại sau trư def _has_kw(topic,title): tl=topic.lower();tt=(title or'').lower() if tl in tt:return True - words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',tl) if len(w)>2 and w.lower() not in _STOP] + words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',tl) if len(w)>2 and w not in _STOP] if not words:return True return any(w in tt for w in words) @@ -351,12 +322,15 @@ def _search_all(topic,limit=36): if i= 2: return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img, 'body': body[:50], 'source': domain, 'url': url} + + # No body found — use OG meta as fallback if title and (summary or og_img): fallback = [] if og_img: fallback.append({'type': 'img', 'src': og_img}) @@ -446,73 +432,98 @@ def _scrape_article_fast(url): if fallback: return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img, 'body': fallback, 'source': domain, 'url': url, 'fallback': True} + + # Got HTML but no body and no OG — return title at least if title: return {'title': _clean(title), 'summary': '', 'og_image': '', 'body': [{'type': 'p', 'text': 'Nội dung đang được tải...'}], 'source': domain, 'url': url, 'fallback': True} - break + + break # Got 200 but no content at all — don't retry other UA except Exception: continue + return None @app.get('/api/article') def api_article_v2(url: str = Query(...)): + """Scrape article and return JSON for VNEWS SPA. Fast, cached, with fallback.""" from urllib.parse import unquote safe_url = unquote(url) + try: + # Check cache first now = time.time() cached = _article_cache.get(safe_url) if cached and now - cached['t'] < _article_cache_ttl: resp = JSONResponse(cached['d']) resp.headers["Cache-Control"] = "public, max-age=1800" return resp + + # Fetch fresh — use fast scraper for ALL sites (simpler, more reliable) data = _scrape_article_fast(safe_url) + if data and data.get('body'): _article_cache[safe_url] = {'d': data, 't': now} resp = JSONResponse(data) resp.headers["Cache-Control"] = "public, max-age=1800" return resp + + # Last resort: try RSS fallback + try: + from main import _fetch_rss_fallback + from urllib.parse import urlparse as _up + rss_data = _fetch_rss_fallback(safe_url, _up(safe_url).netloc) + if rss_data and rss_data.get('title'): + body = [] + if rss_data.get('og_image'): + body.append({'type': 'img', 'src': rss_data['og_image']}) + if rss_data.get('summary'): + sentences = re.split(r'(?<=[.!?])\s+', rss_data['summary']) + for s in sentences[:10]: + if len(s.strip()) > 20: + body.append({'type': 'p', 'text': s.strip()}) + if body: + result = { + 'title': rss_data['title'], 'summary': rss_data['summary'][:500], + 'og_image': rss_data.get('og_image', ''), 'body': body[:50], + 'source': 'rss', 'url': safe_url, 'fallback': True, 'rss': True + } + _article_cache[safe_url] = {'d': result, 't': now} + resp = JSONResponse(result) + resp.headers["Cache-Control"] = "public, max-age=600" + return resp + except Exception: + pass + result = {'error': 'Không đọc được', 'url': safe_url} resp = JSONResponse(result) resp.headers["Cache-Control"] = "public, max-age=60" return resp except Exception as e: - return JSONResponse({'error': f'Server error: {str(e)[:100]}', 'url': safe_url}, status_code=200) + import traceback + tb = traceback.format_exc() + return JSONResponse({'error': f'Server error: {str(e)[:100]}', 'trace': tb[-500:], 'url': safe_url}, status_code=200) _hot_cache={'t':0,'d':[]} -def _parse_feed(feed_url): - try: - r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=4) - r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml') - out=[] - for item in soup.find_all('item')[:12]: - t=item.find('title') - title=_clean(t.get_text() if t else '') - if title: out.append(title) - return out - except Exception: - return [] def _get_hot_topics(): now=time.time() if _hot_cache['d'] and now-_hot_cache['t']<600:return _hot_cache['d'] - feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss'] - titles=[] - with ThreadPoolExecutor(8) as ex: - futs=[ex.submit(_parse_feed,f) for f in feeds] - for f in as_completed(futs,timeout=5): - try: - for t in f.result(): titles.append(t) - except Exception: pass freq={};display={} - for title in titles: - if not title: continue - title=re.sub(r'\s*[-|].*$','',title) - words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP] - if len(words)<2:continue - for n in(3,4,2): - for i in range(max(0,len(words)-n+1)): - phrase=' '.join(words[i:i+n]) - if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase + feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss'] + for feed_url in feeds: + try: + r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=6);r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml') + for item in soup.find_all('item')[:12]: + title=_clean(item.find('title').get_text() if item.find('title') else '') + if not title:continue + title=re.sub(r'\s*[-|].*$','',title);words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP] + if len(words)<2:continue + for n in(3,4,2): + for i in range(max(0,len(words)-n+1)): + phrase=' '.join(words[i:i+n]) + if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase + except:continue ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True);topics=[];seen=set() for key,count in ranked: is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen) @@ -539,209 +550,15 @@ def _ht(topic:str=Query(...),page:int=Query(default=0)): items=_search_all(topic,36);per_page=8;start=page*per_page;end=start+per_page return JSONResponse({'sources':items[start:end],'topic':topic,'page':page,'has_more':end - - - - -{_clean(safe_title)} - - - - - - - -''' - for s in slides: - img_src = s.get('image', '') - if img_src and ('cdnphoto.dantri' in img_src or 'refooty' in img_src or 'vnexpress' in img_src or 'vcdn' in img_src): - img_tag = f'' - else: - img_tag = f'' if img_src else '' - h += f'
Slide {s.get("index",1)}/{len(slides)}
{img_tag}

{_clean(s.get("text",""))}

' - h += '' - return HTMLResponse(h) - -def _render_video_page(post, safe_title, safe_img, safe_url): - video_url = post.get('video', '') - # Use text for description if available - description = _clean((post.get('text') or '')[:200]) or "Tin tức tóm tắt, AI rewrite, World Cup 2026" - - # Build canonical URL preserving original query format if url was provided - if safe_url and safe_url != '/': - canonical_url = f"{SPACE}/s?url={quote(safe_url)}&title={quote(safe_title[:100])}" - else: - canonical_url = f"{SPACE}/s?post_id={post.get('id') or ''}" - - h = f''' - - - - -{_clean(safe_title)} - - - - - - - - - - -
- -
{_clean(safe_title)}
-
-''' - return HTMLResponse(h) - -@app.get('/s/{slug}') -async def _sh_slug(slug: str, request: Request, url: str = '', title: str = '', img: str = ''): - """SEO-friendly share endpoint with slug in URL path. - Shows slide content when slug matches a wall post ID, otherwise redirects. - """ - safe_title = _clean(title) if title else 'VNEWS - Tin tức' - safe_img = _clean(img) if img else '' - safe_url = _clean(url) if url else '/' - - # Try to find post by slug (post ID) - post = None - try: - if slug and len(slug) > 5: # Likely a post ID - posts = _load_wall_posts() - for p in posts: - if p.get('id') == slug: - post = p - safe_title = p.get('title', safe_title) or safe_title - safe_img = p.get('img', safe_img) or safe_img - safe_url = p.get('url', safe_url) or safe_url - break - except: - pass - - if post and post.get('slides'): - return _render_slides_page(post, safe_title, safe_img, safe_url) - - if post and post.get('video'): - return _render_video_page(post, safe_title, safe_img, safe_url) - - # Otherwise redirect - return HTMLResponse(f''' - - - - -{_clean(safe_title)} - - - - - - -''') - @app.get('/s') -async def _sh(url:str='',title:str='',img:str='',post_id:str=''): - safe_title = _clean(title) if title else 'VNEWS - Tin tức' - safe_img = _clean(img) if img else '' - safe_url = _clean(url) if url else '/' - - # Try to find wall post by post_id or URL (prioritize posts with slides/video) - post = None - try: - posts = _load_wall_posts() - if post_id: - for p in posts: - if p.get('id') == post_id: - post = p - safe_title = p.get('title', safe_title) or safe_title - safe_img = p.get('img', safe_img) or safe_img - safe_url = p.get('url', safe_url) or safe_url - break - elif url: - # Find matching URL - prioritize posts with slides or video - for p in posts: - if p.get('url') == url and p.get('slides'): - post = p - safe_title = p.get('title', safe_title) or safe_title - safe_img = p.get('img', safe_img) or safe_img - safe_url = p.get('url', safe_url) or safe_url - break - if not post: - # Fallback: find any matching URL - for p in posts: - if p.get('url') == url: - post = p - safe_title = p.get('title', safe_title) or safe_title - safe_img = p.get('img', safe_img) or safe_img - safe_url = p.get('url', safe_url) or safe_url - break - except: - pass - - if post and post.get('slides'): - return _render_slides_page(post, safe_title, safe_img, safe_url) - - if post and post.get('video'): - return _render_video_page(post, safe_title, safe_img, safe_url) - - # Fallback: redirect to original URL - return HTMLResponse(f''' - - - - -{safe_title} - - - - - - -''') - -from wc2026_scraper import scrape_summary,scrape_fixtures,scrape_standings,scrape_stats,scrape_wc_news,scrape_road_to_wc,get_wc2026_all,scrape_history,scrape_h2h,scrape_lineups,scrape_match_detail +async def _sh(url:str='',title:str='',img:str=''):return HTMLResponse(f'') + +from wc2026_scraper import(scrape_summary,scrape_fixtures,scrape_standings,scrape_stats,scrape_wc_news,scrape_road_to_wc,get_wc2026_all,scrape_history,scrape_h2h,scrape_lineups,scrape_match_detail) +# === XEMLAIBONGDA PROXY (CORS workaround for WC highlights) === _xlb_cache = {} _xlb_lock = threading.Lock() @@ -879,294 +696,54 @@ async def _pc(request:Request): with _il:idb=_lj(IF);idb.setdefault(v,{'views':0,'likes':0,'comments':0});idb[v]['comments']=len(cms);_sj(IF,idb) return JSONResponse({'comments':cms}) +# ===== WALL / SHORT AI ENDPOINTS ===== + def _load_wall_posts(): + """Load wall posts from JSON file.""" with _wl_lock: - posts = _lj(WALL_FILE) - if not isinstance(posts, list): - posts = [] - return posts + return _lj(WALL_FILE) def _save_wall_posts(posts): + """Save wall posts to JSON file.""" with _wl_lock: _sj(WALL_FILE, posts) -def _post_has_valid_image(p): - """A wall post is considered to have a 'valid' (non-placeholder) image if - at least one of the following is a real, non-placeholder URL: - - p.img (main thumbnail) - - p.short_thumb (generated by Short AI / oEmbed fallback) - - p.images[] (fallback image list) - - p.slides[].image (designed slide images) - - Auto posts that failed metadata enrichment — e.g. pollinations.ai - AI-generated placeholder images, vnexpress logo placeholders, or empty/None - fields — are flagged as having NO valid image so they can be filtered out - of the wall (they show the wrong picture vs the original article).""" - PLACEHOLDER_MARKERS = ( - "pollinations.ai", # AI-generated editorial illustration placeholder - "logo_default.jpg", # vnexpress placeholder logo - "logo.png", - "/logo", - "placeholder", - "no-image", - "blank", - ) - - def _is_placeholder(url): - if not url or not isinstance(url, str): - return True - u = url.strip().lower() - if not u or u.startswith(("about:blank", "data:")): - return True - return any(m in u for m in PLACEHOLDER_MARKERS) - - def _is_real(url): - return bool(url) and not _is_placeholder(url) - - img = p.get("img") - if _is_real(img): - return True - thumb = p.get("short_thumb") - if _is_real(thumb): - return True - images = p.get("images") or [] - if any(_is_real(x) for x in images): - return True - slides = p.get("slides") or [] - if any(_is_real(s.get("image")) for s in slides if isinstance(s, dict)): - return True - return False - -def _post_has_slide_info(p): - """True if the post has usable slide/rewrite AI content: - - designed slides with text+image, OR - - a generated short video (/api/ai/short-file/) with a real thumbnail, OR - - an oEmbed iframe (YouTube/TikTok/Facebook embed) with a real thumbnail - (Short HOT fallback case — embeddable, viewable as a slide).""" - slides = p.get("slides") or [] - if any((s.get("image") or "").strip() and (s.get("text") or "").strip() for s in slides if isinstance(s, dict)): - return True - # Generated short video with a real thumbnail - vid = (p.get("video") or "").strip() - if vid and "api/ai/short-file/" in vid and (p.get("short_thumb") or "").strip(): - return True - # oEmbed iframe (YouTube/TikTok/FB) with a valid image -> viewable embed slide - if vid and ("youtube.com/embed" in vid or "tiktok.com/embed" in vid or - "facebook.com/plugins" in vid or "instagram.com" in vid or - "twitter.com" in vid or "x.com" in vid or "player.vimeo" in vid): - if _post_has_valid_image(p): - return True - return False - -def _is_auto_post(p): - """Auto-generated wall posts: Short HOT, oEmbed fallback, FPT auto, - slide-rewrite AI (auto_rewrite), topic auto-summary (topic_article), or - posts with no meaningful source / created by the AI pipeline.""" - kind = (p.get("kind") or "").strip().lower() - source = (p.get("source") or "").strip().lower() - return kind in ("hot_short", "slide_summary", "auto_rewrite", "topic_article", "summary") or \ - source in ("hot_short", "fptplay", "ai", "auto_rewrite", "topic_article") - -def _created_ts(p): - """Best-effort creation timestamp. Posts created by the AI pipeline often - store `created` as a millisecond epoch; some older posts (auto_rewrite) - omit it and instead embed the timestamp in the `id` (Unix ms). Fall back - to 0 so sorting never crashes.""" - for field in ("created", "ts"): - v = p.get(field) - if v is not None: - try: - return int(v) - except (ValueError, TypeError): - pass - # try to derive from id (numeric prefix = Unix ms timestamp) - pid = str(p.get("id", "")).strip() - if pid.isdigit() and len(pid) >= 12: - try: - return int(pid) - except (ValueError, TypeError): - pass - return 0 - -def _wall_posts_for_view(): - """Apply the user-facing wall filtering + sorting: - 1) Hide auto-generated posts that have a wrong/placeholder image (no - valid image at all — pollinations.ai, vnexpress logo, empty) or have - no slide-AI content (no slides / no short-video / no embed). These - are the 'bài đăng tự động bị sai ảnh / ko có thông tin slide AI' that - don't match the original article. This filter is VIEW-ONLY — it does - NOT delete from the persistent store. - 2) Sort newest-first by creation timestamp, mixing all sources (FPT, - slide rewrite AI, etc.) without source separation.""" - posts = _load_wall_posts() - if not posts: - return [] - filtered = [] - for p in posts: - if not isinstance(p, dict): - continue - if _is_auto_post(p) and not (_post_has_valid_image(p) and _post_has_slide_info(p)): - # auto post with a wrong/placeholder/missing image or no slide AI - # content -> hide from the wall (non-destructive) - continue - filtered.append(p) - filtered.sort(key=_created_ts, reverse=True) - return filtered - -@app.get("/api/ai/wall") -def api_ai_wall(): - return JSONResponse({"posts": _wall_posts_for_view()}) - - -# ===== YOUTUBE RSS FEED PROXY (avoids browser CORS) ===== -_YT_FEED_CACHE = {} -_YT_FEED_TTL = 600 # 10 minutes - - -def _fetch_youtube_rss_feed(channel_id: str = "UC4LvrpNXujjbGOS4RDvr41g"): - """Fetch & parse the YouTube RSS feed server-side. - - The browser cannot reach youtube.com directly (CORS / network block). - We try two sources in order: - 1) Direct feed XML (works if the server has outbound YouTube access) - 2) rss2json.com proxy (always works — it fetches the feed server-side) - Cached for _YT_FEED_TTL seconds.""" - now = time.time() - cached = _YT_FEED_CACHE.get(channel_id) - if cached and now - cached.get('_ts', 0) < _YT_FEED_TTL: - return cached.get('videos', []) - out = [] - try: - # Attempt 1: direct YouTube RSS - feed_url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}" - try: - r = req.get(feed_url, headers=HEADERS, timeout=10) - if r.status_code == 200 and r.text.strip(): - soup = BeautifulSoup(r.text, 'xml') - for e in soup.find_all('entry'): - out.extend(_parse_yt_entry(e)) - except Exception as ex2: - print(f"[yt-feed] direct fetch failed, trying rss2json: {ex2}") - # Attempt 2: rss2json proxy (fallback / always) - if not out: - rss2json_url = f"https://api.rss2json.com/v1/api.json?rss_url={feed_url}" - r2 = req.get(rss2json_url, headers=HEADERS, timeout=15) - if r2.status_code == 200: - data = r2.json() - if data.get('status') == 'ok': - for item in data.get('items', []): - vid_m = re.search(r'([a-zA-Z0-9_-]{11})', item.get('id', '')) - if not vid_m: - continue - vid = vid_m.group(1) - title = item.get('title', 'Video') or 'Video' - # prefer the youtube watch URL, fallback to item link - link = item.get('link', '') - if '/watch' not in link: - link = f'https://www.youtube.com/watch?v={vid}' - # thumbnail: prefer media:thumbnail, fallback to youtube hqdefault - thumb = item.get('thumbnail', f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg') - published = item.get('pubDate', '') - out.append({ - 'id': 'yt-' + vid, - 'videoId': vid, - 'title': title.strip()[:200], - 'link': link, - 'img': thumb, - 'published': published, - 'source': 'yt-feed' - }) - if out: - _YT_FEED_CACHE[channel_id] = {'_ts': now, 'videos': out} - return out - except Exception as ex: - print(f"[yt-feed] server error: {ex}") - return [] - - -def _parse_yt_entry(e): - """Parse a single YouTube RSS element into a video dict.""" - out = [] - try: - id_el = e.find('id') - if not id_el: - return out - vid_m = re.search(r'([a-zA-Z0-9_-]{11})', id_el.get_text('')) - if not vid_m: - return out - vid = vid_m.group(1) - title_el = e.find('title') - title = title_el.get_text('') if title_el else 'Video' - link_el = e.find('link', {'rel': 'alternate'}) - link = link_el.get('href') if link_el else f'https://www.youtube.com/watch?v={vid}' - pub_el = e.find('published') - published = pub_el.get_text('') if pub_el else '' - ns = {'media': 'http://search.yahoo.com/mrss/'} - thumb_el = e.find('media:thumbnail', ns) - thumb_url = thumb_el.get('url') if thumb_el else f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg' - out.append({ - 'id': 'yt-' + vid, - 'videoId': vid, - 'title': title.strip()[:200], - 'link': link, - 'img': thumb_url, - 'published': published, - 'source': 'yt-feed' - }) - except Exception as ex: - print(f"[yt-feed] parse entry error: {ex}") - return out - - -@app.get("/api/yt/feed") -def api_yt_feed(channel_id: str = Query(default="UC4LvrpNXujjbGOS4RDvr41g")): - """Server-side YouTube RSS feed fetcher — avoids browser CORS. - Returns parsed feed videos as JSON for the Tường AI wall.""" - videos = _fetch_youtube_rss_feed(channel_id) - return JSONResponse({"videos": videos, "count": len(videos)}) -@app.post("/api/ai/short/{post_id}") -def api_ai_short(post_id: str): - """Generate a short video for a wall post. - Returns clear JSON error when video not available yet. - """ - post_id_s = str(post_id) - posts = _load_wall_posts() - if not isinstance(posts, list): - posts = [] - post = None - for p in posts: - pid = str(p.get("id", "")) - if pid == post_id_s or pid.startswith(post_id_s): - post = p - break - if not post: - return JSONResponse({"error": "Không tìm thấy bài viết", "post_id": post_id}, status_code=404) - if post.get("video"): - return JSONResponse({"video": post["video"], "post": post}) - return JSONResponse({"error": "Chưa có video cho bài này. Vui lòng upload video trước."}, status_code=409) - @app.get('/api/wall') def api_wall(): - posts = _wall_posts_for_view() + """Get all wall posts.""" + posts = _load_wall_posts() if not posts: + # Return empty list, not error return JSONResponse({"posts": []}) return JSONResponse({"posts": posts}) @app.post('/api/wall') async def api_wall_post(request: Request): + """ + Create a wall post. Supports: + - JSON body: {title, text, img, source} + - Multipart form: title, text, source + video file upload + """ content_type = request.headers.get('content-type', '') + + # Handle multipart upload (video file) if 'multipart/form-data' in content_type: try: form = await request.form() except Exception as e: return JSONResponse({"error": f"Form parse error: {str(e)}"}, status_code=400) + title = form.get('title', 'Video mới') or 'Video mới' text = form.get('text', '') or '' source = form.get('source', 'vtv_recorder') or 'vtv_recorder' video_file = form.get('video') + post_id = str(uuid.uuid4())[:12] video_url = None + + # Save video file if provided if video_file and hasattr(video_file, 'filename') and video_file.filename: + # Determine extension fname = video_file.filename.lower() if fname.endswith('.mp4'): ext = '.mp4' @@ -1174,21 +751,31 @@ async def api_wall_post(request: Request): ext = '.webm' else: ext = '.webm' + video_filename = f"wall_{post_id}{ext}" video_path = os.path.join(WALL_VIDEO_DIR, video_filename) + try: + # Read file content content = await video_file.read() if not content: return JSONResponse({"error": "Empty video file"}, status_code=400) + + # Save to disk with open(video_path, 'wb') as f: f.write(content) + file_size_mb = len(content) / 1024 / 1024 if file_size_mb > 50: os.remove(video_path) return JSONResponse({"error": f"Video quá lớn ({file_size_mb:.1f}MB). Tối đa 50MB."}, status_code=400) + + # URL to access the video video_url = f"/api/wall/video/{video_filename}" except Exception as e: return JSONResponse({"error": f"Lỗi lưu video: {str(e)}"}, status_code=500) + + # Create post post = { "id": post_id, "title": title[:200], @@ -1200,87 +787,55 @@ async def api_wall_post(request: Request): "created": int(time.time()), "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()), } + + # Save to wall posts = _load_wall_posts() if not isinstance(posts, list): posts = [] posts.insert(0, post) + # Keep max 200 posts posts = posts[:200] _save_wall_posts(posts) + return JSONResponse({"post": post, "ok": True}) + + # Handle JSON body (text-only post) try: body = await request.json() except: body = {} + title = body.get('title', 'Bài mới') or 'Bài mới' text = body.get('text', '') or '' img = body.get('img', None) source = body.get('source', 'user') or 'user' - incoming_id = body.get('id') or '' # allow client to (re)publish an existing post by id + post_id = str(uuid.uuid4())[:12] - # Preserve slide-design post fields (slides, kind, url, images, video, voice, etc.) - # so that "Thiết kế ảnh" -> "Đăng lên Tường AI" keeps the selected slides. - slides = body.get('slides') - kind = body.get('kind') or 'user' - post_url = body.get('url') or '' - images = body.get('images') or [] - video = body.get('video') - voice = body.get('voice') or '' - emotion = body.get('emotion') or '' - language = body.get('language') or '' post = { "id": post_id, "title": title[:200], "text": text[:2000], "source": source, - "video": video, + "video": None, "img": img, - "images": images[:10], - "url": post_url, - "kind": kind, - "slides": slides if slides is not None else None, - "voice": voice, - "emotion": emotion, - "language": language, + "images": [], "created": int(time.time()), "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()), } - # Remove keys with None so the JSON stays lean (but keep structure) - post = {k: v for k, v in post.items() if v is not None} + posts = _load_wall_posts() if not isinstance(posts, list): posts = [] - # If the client re-publishes (e.g. designer "Lưu & Đăng lên Tường AI" sends - # the original post id), update that post in place instead of creating a - # duplicate. This keeps the homepage wall clean (no duplicate slide posts). - updated = False - if incoming_id: - for _p in posts: - if str(_p.get('id')) == str(incoming_id): - # Merge new fields into the existing post, but PRESERVE the - # original id (and created timestamp) so the post identity is - # stable and no duplicate is created. - preserve_id = _p.get('id') - preserve_created = _p.get('created') - preserve_created_str = _p.get('created_str') - _p.update(post) - _p['id'] = preserve_id - if preserve_created is not None: _p['created'] = preserve_created - if preserve_created_str is not None: _p['created_str'] = preserve_created_str - updated = True - post = _p # return the merged post (with stable id) to the client - break - if not updated: - posts.insert(0, post) - else: - # Move the updated post to the top so it re-appears at the top of the wall. - posts = [p for p in posts if str(p.get('id')) != str(incoming_id)] - posts.insert(0, post) + posts.insert(0, post) posts = posts[:200] _save_wall_posts(posts) + return JSONResponse({"post": post, "ok": True}) @app.get('/api/wall/video/{filename}') def api_wall_video(filename: str): + """Serve a wall video file.""" + # Security: prevent path traversal if '..' in filename or '/' in filename: return Response(status_code=403) video_path = os.path.join(WALL_VIDEO_DIR, filename) @@ -1290,64 +845,16 @@ def api_wall_video(filename: str): media_type = 'video/mp4' if ext == '.mp4' else 'video/webm' return FileResponse(video_path, media_type=media_type) -WALL_IMG_DIR = os.path.join(DATA_DIR, 'wall_imgs') -os.makedirs(WALL_IMG_DIR, exist_ok=True) - -@app.post('/api/wall/img') -async def api_wall_img(request: Request): - """Upload a designed slide image (PNG) and return a served URL.""" - global WALL_IMG_DIR - try: - form = await request.form() - f = form.get('file') - if not f or not hasattr(f, 'filename') or not f.filename: - return JSONResponse({"error": "Thiếu file ảnh"}, status_code=400) - ext = os.path.splitext(f.filename)[1].lower() - if ext not in ('.png', '.jpg', '.jpeg', '.webp'): - ext = '.png' - img_id = str(uuid.uuid4())[:12] - fname = f"wallimg_{img_id}{ext}" - fpath = os.path.join(WALL_IMG_DIR, fname) - content = await f.read() - if not content: - return JSONResponse({"error": "File rỗng"}, status_code=400) - if len(content) > 15 * 1024 * 1024: - return JSONResponse({"error": "Ảnh quá lớn (>15MB)"}, status_code=400) - with open(fpath, 'wb') as fh: - fh.write(content) - post_id = (form.get('post_id') or '').strip() - if post_id: - posts = _load_wall_posts() - if isinstance(posts, list): - for p in posts: - if str(p.get('id')) == str(post_id): - p['img'] = f"/api/wall/img/{fname}" - break - _save_wall_posts(posts) - url = f"/api/wall/img/{fname}" - return JSONResponse({"ok": True, "url": url, "img_id": img_id}) - except Exception as e: - return JSONResponse({"error": f"Lỗi upload ảnh: {str(e)[:150]}"}, status_code=500) - - -@app.get('/api/wall/img/{fname}') -def api_wall_img_file(fname: str): - if '..' in fname or '/' in fname: - return Response(status_code=403) - img_path = os.path.join(WALL_IMG_DIR, fname) - if not os.path.exists(img_path): - return Response(status_code=404) - ext = os.path.splitext(fname)[1].lower() - media_type = 'image/png' if ext == '.png' else ('image/jpeg' if ext in ('.jpg', '.jpeg') else 'image/webp') - return FileResponse(img_path, media_type=media_type) - @app.delete('/api/wall/{post_id}') def api_wall_delete(post_id: str): + """Delete a wall post and its video.""" posts = _load_wall_posts() if not isinstance(posts, list): return JSONResponse({"error": "No posts"}, status_code=404) + for i, p in enumerate(posts): if p.get('id') == post_id: + # Delete video file if exists if p.get('video'): video_name = p['video'].split('/')[-1] video_path = os.path.join(WALL_VIDEO_DIR, video_name) @@ -1356,1052 +863,8 @@ def api_wall_delete(post_id: str): posts.pop(i) _save_wall_posts(posts) return JSONResponse({"ok": True}) - return JSONResponse({"error": "Post not found"}, status_code=404) - -@app.post('/api/wall/cleanup') -def api_wall_cleanup(): - """Permanently remove auto-generated wall posts that are completely - unrecoverable (no valid image, no video, no slides, no embed — i.e. a - post that can never be displayed), and sort the remaining posts - newest-first. Returns the number of removed posts and the new count. - - NOTE: Conservative — auto posts that HAVE a valid image or a video/embed - are preserved (they remain visible at view time via - _wall_posts_for_view). Only use this to purge genuine garbage.""" - posts = _load_wall_posts() - if not isinstance(posts, list): - return JSONResponse({"error": "No posts"}, status_code=404) - before = len(posts) - kept = [] - removed = [] - for p in posts: - if not isinstance(p, dict): - removed.append(str(p)) - continue - if _is_auto_post(p) and not _post_has_valid_image(p) and not _post_has_slide_info(p) and not (p.get("video") or "").strip(): - removed.append(p.get('id', '?')) - else: - kept.append(p) - # sort newest-first - kept.sort(key=_created_ts, reverse=True) - _save_wall_posts(kept) - return JSONResponse({ - "ok": True, - "before": before, - "removed_count": before - len(kept), - "removed_ids": removed, - "after": len(kept), - }) - -# ===== LANGUAGE & EMOTION DETECTION ===== -import random as _random2 -from urllib.parse import quote as _quote2 - -_UA_RW = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', 'Accept-Language': 'vi-VN,vi;q=0.9'} - -# Unique character markers for language detection -_UNIQUE_CHARS = { - 'vietnamese': set('đăâêôơưàảãạáằẳẵặắầẩẫậấèẻẽẹéềễểệếìỉĩịíòỏõọóồổỗộốờởỡợớùủũụúừửữựứỳỷỹỵý'), - 'spanish': set('ñáéíóúü¿¡'), - 'portuguese': set('ãõçáéíóúâêôà'), -} - -_STOPWORDS = { - 'english': {'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with', 'to', 'for', 'of', 'not', 'no', 'can', 'had', 'have', 'has', 'was', 'were', 'are', 'be', 'been', 'this', 'that', 'it', 'he', 'she', 'they', 'his', 'her', 'my', 'your', 'our', 'we', 'you', 'i'}, - 'vietnamese': {'là', 'của', 'và', 'có', 'được', 'cho', 'không', 'với', 'này', 'đó', 'từ', 'trong', 'đã', 'sẽ', 'một', 'các', 'những', 'về', 'tại', 'người', 'năm', 'đến', 'ra', 'lại', 'như', 'khi', 'để', 'rất', 'cũng', 'mà', 'nếu', 'sau', 'trên', 'theo', 'vì', 'do', 'nên', 'thì', 'mình', 'tôi', 'bạn', 'anh', 'chị', 'em'}, - 'portuguese': {'de', 'um', 'que', 'e', 'do', 'da', 'em', 'para', 'com', 'não', 'uma', 'os', 'no', 'se', 'na', 'por', 'mais', 'as', 'dos', 'como', 'mas', 'ao', 'ele', 'das', 'tem', 'seu', 'sua', 'ou', 'quando', 'muito', 'nos', 'já', 'eu', 'também', 'só', 'pelo', 'pela', 'até', 'isso', 'ela', 'entre', 'depois', 'sem', 'mesmo', 'aos', 'são', 'está', 'ter', 'ser', 'foi', 'era', 'há', 'estão', 'você', 'nós', 'eles', 'elas'}, - 'spanish': {'de', 'que', 'el', 'en', 'y', 'a', 'los', 'del', 'se', 'las', 'por', 'un', 'para', 'con', 'no', 'una', 'su', 'al', 'es', 'lo', 'como', 'más', 'pero', 'sus', 'le', 'ya', 'o', 'fue', 'este', 'ha', 'si', 'porque', 'esta', 'son', 'entre', 'está', 'cuando', 'muy', 'sin', 'sobre', 'ser', 'también', 'me', 'hasta', 'hay', 'donde', 'han', 'quien', 'están', 'desde', 'todo', 'nos', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'fueron', 'ese', 'eso', 'ante', 'ellos', 'yo', 'tú', 'él', 'ella', 'nosotros', 'usted', 'ustedes'}, -} - -def detect_language(text): - """Detect language from text content using stopword + character analysis.""" - if not text: - return 'vietnamese' - text_lower = text.lower() - text_chars = set(text_lower) - - # Strong signal: Vietnamese unique characters - vn_chars = len(text_chars & _UNIQUE_CHARS['vietnamese']) - if vn_chars >= 2: - return 'vietnamese' - - # Spanish unique chars (ñ, ¿, ¡) - es_chars = len(text_chars & _UNIQUE_CHARS['spanish']) - pt_chars = len(text_chars & _UNIQUE_CHARS['portuguese']) - - # Stopword scoring - words = set(re.findall(r'\b\w+\b', text_lower)) - scores = {} - for lang, stops in _STOPWORDS.items(): - scores[lang] = len(words & stops) / max(len(stops), 1) - - # Disambiguate Portuguese vs Spanish - pt_markers = {'não', 'pelo', 'pela', 'isso', 'há', 'estão', 'num', 'numa', 'tenho', 'posso', 'você', 'nós', 'eles', 'elas', 'também', 'muito', 'já', 'só', 'até', 'entre', 'depois', 'sem', 'mesmo', 'aos', 'serão'} - es_markers = {'pero', 'está', 'están', 'porque', 'también', 'hasta', 'donde', 'quien', 'fue', 'son', 'fueron', 'ese', 'eso', 'ante', 'ellos', 'ella', 'nosotros', 'usted', 'ustedes', 'tú', 'él', 'desde', 'todo', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'fueron'} - - pt_overlap = len(words & pt_markers) - es_overlap = len(words & es_markers) - - if scores.get('portuguese', 0) > 0 and pt_overlap > es_overlap: - return 'portuguese' - if scores.get('spanish', 0) > 0 and es_overlap > pt_overlap: - return 'spanish' - if scores.get('english', 0) > 0.15: - return 'english' - - best = max(scores, key=scores.get) - return best if scores[best] > 0.05 else 'vietnamese' - -# Emotion keyword-based detection -_EMOTION_KEYWORDS = { - 'happy': { - 'en': ['happy', 'joy', 'wonderful', 'great', 'amazing', 'fantastic', 'love', 'excellent', 'beautiful', 'glad', 'delighted', 'pleased', 'cheerful', 'celebrate', 'victory', 'win', 'success'], - 'pt': ['feliz', 'alegria', 'maravilhoso', 'ótimo', 'incrível', 'fantástico', 'amor', 'excelente', 'lindo', 'contente', 'encantado', 'vitória', 'sucesso'], - 'es': ['feliz', 'alegria', 'maravilloso', 'genial', 'increíble', 'fantástico', 'amor', 'excelente', 'hermoso', 'contento', 'encantado', 'victoria', 'éxito'], - 'vi': ['vui', 'hạnh phúc', 'tuyệt vời', 'tuyệt', 'ý nghĩa', 'đẹp', 'thích', 'yêu', 'vui vẻ', 'hân hoan', 'phấn khích', 'chiến thắng', 'thành công'], - }, - 'sad': { - 'en': ['sad', 'unhappy', 'terrible', 'awful', 'horrible', 'miserable', 'depressed', 'grief', 'sorrow', 'tragic', 'unfortunate', 'painful', 'death', 'die', 'kill'], - 'pt': ['triste', 'infeliz', 'terrível', 'horrível', 'miserável', 'deprimido', 'dor', 'trágico', 'infelizmente', 'penoso', 'morte', 'morrer'], - 'es': ['triste', 'infeliz', 'terrible', 'horrible', 'miserable', 'deprimido', 'dolor', 'trágico', 'desafortunado', 'penoso', 'muerte', 'morir'], - 'vi': ['buồn', 'không vui', 'tồi tệ', 'kinh khủng', 'đau khổ', 'đau buồn', 'bi thương', 'khốn nạn', 'đau đớn', 'thảm họa', 'chết', 'mất'], - }, - 'excited': { - 'en': ['excited', 'thrilling', 'amazing', 'wow', 'incredible', 'unbelievable', 'awesome', 'exhilarating', 'electrifying', 'breathtaking', 'breakthrough', 'record'], - 'pt': ['animado', 'emocionante', 'incrível', 'impressionante', 'sensacional', 'eletrizante', 'empolgante', 'recorde'], - 'es': ['emocionante', 'increíble', 'impresionante', 'sensacional', 'electrizante', 'emocionado', 'entusiasmado', 'récord'], - 'vi': ['hào hứng', 'phấn khích', 'thú vị', 'tuyệt cú mèo', 'đỉnh cao', 'ngoạn mục', 'sục sôi', 'kỷ lục', 'đột phá'], - }, - 'humorous': { - 'en': ['funny', 'hilarious', 'joke', 'laugh', 'comedy', 'humor', 'amusing', 'witty', 'sarcastic', 'ironic', 'ridiculous', 'absurd', 'lol', 'haha'], - 'pt': ['engraçado', 'hilário', 'piada', 'rir', 'comédia', 'humor', 'divertido', 'irônico', 'ridículo', 'absurdo', 'kkk'], - 'es': ['gracioso', 'hilarante', 'broma', 'risa', 'comedia', 'humor', 'divertido', 'irónico', 'ridículo', 'absurdo', 'jaja'], - 'vi': ['hài hước', 'buồn cười', 'đùa', 'cười', 'hài', 'vui nhộn', 'hóm hỉnh', 'mỉa mai', 'lố bịch', 'vô lý', 'haha'], - }, - 'serious': { - 'en': ['serious', 'critical', 'important', 'urgent', 'severe', 'grave', 'significant', 'crucial', 'vital', 'essential', 'alarming', 'concerning', 'crisis', 'war', 'conflict'], - 'pt': ['sério', 'crítico', 'importante', 'urgente', 'grave', 'significativo', 'crucial', 'vital', 'essencial', 'preocupante', 'crise', 'guerra', 'conflito'], - 'es': ['serio', 'crítico', 'importante', 'urgente', 'grave', 'significativo', 'crucial', 'vital', 'esencial', 'preocupante', 'crisis', 'guerra', 'conflicto'], - 'vi': ['nghiêm trọng', 'quan trọng', 'khẩn cấp', 'nghiêm túc', 'đáng kể', 'thiết yếu', 'cần thiết', 'báo động', 'lo ngại', 'khủng hoảng', 'chiến tranh', 'xung đột'], - }, -} - -def detect_emotion(text, language='vietnamese'): - """Detect emotion from text using keyword matching.""" - if not text: - return 'neutral' - text_lower = text.lower() - - scores = {} - for emotion, lang_keywords in _EMOTION_KEYWORDS.items(): - keywords = lang_keywords.get(language, lang_keywords.get('en', [])) - score = sum(1 for kw in keywords if kw in text_lower) - scores[emotion] = score - - if max(scores.values()) == 0: - return 'neutral' - - return max(scores, key=scores.get) - -def detect_language_and_emotion(title, text): - """Detect both language and emotion from article content.""" - combined = f"{title} {text}" - lang = detect_language(combined) - emotion = detect_emotion(combined, lang) - return lang, emotion - -# Voice selection based on language and emotion (using MultilingualNeural voices) -VOICE_BY_LANG_EMOTION = { - 'vietnamese': { - 'happy': ('vi-VN-HoaiMyNeural', 'vui'), - 'sad': ('vi-VN-NamMinhNeural', 'buồn'), - 'excited': ('vi-VN-HoaiMyNeural', 'hào hứng'), - 'humorous': ('vi-VN-HoaiMyNeural', 'vui'), - 'serious': ('vi-VN-NamMinhNeural', 'nghiêm túc'), - 'neutral': ('vi-VN-HoaiMyNeural', 'trung_tinh'), - }, - 'portuguese': { - 'happy': ('pt-BR-ThalitaMultilingualNeural', 'feliz'), - 'sad': ('pt-BR-ThalitaMultilingualNeural', 'triste'), - 'excited': ('pt-BR-ThalitaMultilingualNeural', 'animado'), - 'humorous': ('pt-BR-ThalitaMultilingualNeural', 'engraçado'), - 'serious': ('pt-BR-ThalitaMultilingualNeural', 'sério'), - 'neutral': ('pt-BR-ThalitaMultilingualNeural', 'neutro'), - }, - 'english': { - 'happy': ('en-US-AndrewMultilingualNeural', 'happy'), - 'sad': ('en-AU-WilliamMultilingualNeural', 'sad'), - 'excited': ('en-US-AndrewMultilingualNeural', 'excited'), - 'humorous': ('en-US-AndrewMultilingualNeural', 'funny'), - 'serious': ('en-AU-WilliamMultilingualNeural', 'serious'), - 'neutral': ('en-US-AndrewMultilingualNeural', 'neutral'), - }, - 'french': { - 'happy': ('fr-FR-VivienneMultilingualNeural', 'heureux'), - 'sad': ('fr-FR-RemyMultilingualNeural', 'triste'), - 'excited': ('fr-FR-VivienneMultilingualNeural', 'excité'), - 'humorous': ('fr-FR-VivienneMultilingualNeural', 'drôle'), - 'serious': ('fr-FR-RemyMultilingualNeural', 'sérieux'), - 'neutral': ('fr-FR-VivienneMultilingualNeural', 'neutre'), - }, - 'german': { - 'happy': ('de-DE-SeraphinaMultilingualNeural', 'glücklich'), - 'sad': ('de-DE-FlorianMultilingualNeural', 'traurig'), - 'excited': ('de-DE-SeraphinaMultilingualNeural', 'aufgeregt'), - 'humorous': ('de-DE-SeraphinaMultilingualNeural', 'lustig'), - 'serious': ('de-DE-FlorianMultilingualNeural', 'ernst'), - 'neutral': ('de-DE-SeraphinaMultilingualNeural', 'neutral'), - }, - 'korean': { - 'happy': ('ko-KR-HyunsuMultilingualNeural', '행복'), - 'sad': ('ko-KR-HyunsuMultilingualNeural', '슬픔'), - 'excited': ('ko-KR-HyunsuMultilingualNeural', '흥분'), - 'humorous': ('ko-KR-HyunsuMultilingualNeural', '유쾌'), - 'serious': ('ko-KR-HyunsuMultilingualNeural', '진지'), - 'neutral': ('ko-KR-HyunsuMultilingualNeural', '중립'), - }, - 'italian': { - 'happy': ('it-IT-GiuseppeMultilingualNeural', 'felice'), - 'sad': ('it-IT-GiuseppeMultilingualNeural', 'triste'), - 'excited': ('it-IT-GiuseppeMultilingualNeural', 'emozionato'), - 'humorous': ('it-IT-GiuseppeMultilingualNeural', 'divertente'), - 'serious': ('it-IT-GiuseppeMultilingualNeural', 'serio'), - 'neutral': ('it-IT-GiuseppeMultilingualNeural', 'neutro'), - }, -} - -# All valid voice IDs (new MultilingualNeural format) -VALID_VOICES = { - 'vi-VN-HoaiMyNeural', 'vi-VN-NamMinhNeural', - 'en-US-AndrewMultilingualNeural', 'en-AU-WilliamMultilingualNeural', - 'pt-BR-ThalitaMultilingualNeural', - 'fr-FR-VivienneMultilingualNeural', 'fr-FR-RemyMultilingualNeural', - 'de-DE-SeraphinaMultilingualNeural', 'de-DE-FlorianMultilingualNeural', - 'ko-KR-HyunsuMultilingualNeural', - 'it-IT-GiuseppeMultilingualNeural', -} - -def get_voice_for_content(title, text, preferred_voice=None): - """Get appropriate voice based on content language and emotion.""" - # Accept the new MultilingualNeural voices directly - if preferred_voice and preferred_voice in VALID_VOICES: - return preferred_voice - - # Also accept old shorthand voice IDs and map them to new format - old_voice_map = { - 'hoaimy': 'vi-VN-HoaiMyNeural', - 'namminh': 'vi-VN-NamMinhNeural', - 'andrew': 'en-US-AndrewMultilingualNeural', - 'jenny': 'en-US-AndrewMultilingualNeural', - 'thalita': 'pt-BR-ThalitaMultilingualNeural', - 'pt_thalita': 'pt-BR-ThalitaMultilingualNeural', - 'pt_francisco': 'pt-BR-ThalitaMultilingualNeural', - 'ela': 'en-US-AndrewMultilingualNeural', - 'es_carlos': 'en-US-AndrewMultilingualNeural', - 'denise': 'fr-FR-VivienneMultilingualNeural', - 'katja': 'de-DE-SeraphinaMultilingualNeural', - 'nanami': 'en-US-AndrewMultilingualNeural', - 'sunhee': 'ko-KR-HyunsuMultilingualNeural', - 'xiaochen': 'en-US-AndrewMultilingualNeural', - } - if preferred_voice and preferred_voice in old_voice_map: - return old_voice_map[preferred_voice] - - lang, emotion = detect_language_and_emotion(title, text) - lang_map = VOICE_BY_LANG_EMOTION.get(lang, VOICE_BY_LANG_EMOTION['vietnamese']) - voice, _ = lang_map.get(emotion, lang_map['neutral']) - return voice - - -def _is_relevant_image(img_url, title, text): - """Check if an image is relevant to the article content.""" - if not img_url: - return False - skip_patterns = ['pixel', 'analytics', 'tracking', '1x1.gif', 'spacer.gif', - 'logo', 'icon', 'avatar', 'emoji', 'smiley', 'sprite', - 'advertisement', 'ad-banner', 'sponsored', 'banner-ads'] - img_lower = img_url.lower() - for p in skip_patterns: - if p in img_lower: - return False - if not any(img_lower.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.webp', '.gif']): - return False - return True - - -def _filter_relevant_images(images, title, text, max_images=8): - """Filter and rank images by relevance to article content.""" - if not images: - return [] - seen = set() - relevant = [] - for img in images: - if img in seen: - continue - seen.add(img) - if _is_relevant_image(img, title, text): - relevant.append(img) - return relevant[:max_images] - - -def _scrape_article_for_rewrite(url): - """Scrape article: extract title, paragraphs, RELEVANT images, OG image.""" - try: - r = req.get(url, headers=_UA_RW, timeout=15, allow_redirects=True) - r.encoding = 'utf-8' - soup = BeautifulSoup(r.text, 'lxml') - for tag in soup.find_all(['script', 'style', 'nav', 'footer', 'aside', 'form']): - tag.decompose() - h1 = soup.find('h1') - ogt = soup.find('meta', property='og:title') - title = (h1.get_text(strip=True) if h1 else '') or (ogt.get('content', '') if ogt else '') - ogi = soup.find('meta', property='og:image') - og_img = ogi.get('content', '') if ogi else '' - if og_img and og_img.startswith('//'): - og_img = 'https:' + og_img - block = None - for sel in ['article', '.singular-content', '.detail-content', '.fck_detail', '.content-detail', '.knc-content', 'main', '.cms-body', '.article__body']: - el = soup.select_one(sel) - if el and len(el.find_all('p')) >= 2: - block = el - break - if not block: - block = soup.body or soup - paragraphs = [] - all_images = [] - seen_imgs = set() - if og_img and og_img not in seen_imgs: - all_images.append(og_img) - seen_imgs.add(og_img) - for el in block.find_all(['p', 'h2', 'h3', 'figure', 'img'], recursive=True): - if el.name == 'p': - t = _clean(el.get_text(strip=True)) - if t and len(t) > 40: - paragraphs.append(t) - elif el.name in ('figure', 'img'): - im = el if el.name == 'img' else el.find('img') - if im: - src = im.get('data-src') or im.get('src') or im.get('data-original') or '' - if src and 'base64' not in src: - if src.startswith('//'): - src = 'https:' + src - if src not in seen_imgs: - all_images.append(src) - seen_imgs.add(src) - # Filter to relevant images only - relevant_images = _filter_relevant_images(all_images, title, ' '.join(paragraphs[:5])) - return {'title': _clean(title), 'paragraphs': paragraphs, 'images': relevant_images, 'og_img': og_img} - except Exception: - return None - - -def _extract_key_points_rw(paragraphs, max_points=5): - r"""Extract key points from paragraphs - extracts ALL sentences, not just first one. - - Fixes: Original regex `^(.+?[.!?])\s` only captured first sentence per paragraph. - Now splits on all sentence boundaries and takes valid sentences until max_points. - """ - points = [] - - for p in paragraphs: - if len(points) >= max_points: - break - - p = _clean(p) - if not p: - continue - - # Split paragraph into sentences using Vietnamese + English punctuation - sentences = re.split(r'(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])', p) - sentences = [s.strip() for s in sentences if s.strip()] - - for sentence in sentences: - if len(points) >= max_points: - break - - # Clean sentence - remove extra whitespace - sentence = _clean(sentence) - - if len(sentence) < 30: - continue - - # Check for duplicates - if any(sentence[:60] in existing for existing in points): - continue - - # Ensure sentence ends with punctuation - if not sentence.endswith(('.', '!', '?')): - sentence = sentence + '.' - - points.append(sentence) - - # If no valid sentences found, take chunks from raw text - if not points: - raw = '\n'.join(paragraphs) - for i in range(0, min(len(raw), max_points * 300), 280): - chunk = _clean(raw[i:i+280]) - if len(chunk) >= 30 and chunk not in points: - points.append(chunk + ('.' if not chunk.endswith('.') else '')) - if len(points) >= max_points: - break - - return points - - -@app.post("/api/rewrite_slide") -async def api_rewrite_slide(request: Request): - """Fast rewrite as SLIDES - no AI needed, instant response.""" - body = await request.json() - url = _clean(body.get("url", "")) - context = body.get("context", "") - preferred_voice = body.get("voice", "") # Accept custom voice selection - if not url and not context: - return JSONResponse({"error": "Cần URL hoặc nội dung"}, status_code=400) - data = None - if url and url.startswith("http"): - data = _scrape_article_for_rewrite(url) - if not data and context: - paragraphs = [_clean(p) for p in context.split('\n') if len(_clean(p)) > 40] - data = {'title': paragraphs[0][:80] if paragraphs else 'Bài viết', 'paragraphs': paragraphs, 'images': [], 'og_img': ''} - if not data or not data.get('paragraphs'): - return JSONResponse({"error": "Không đọc được bài viết"}, status_code=422) - points = _extract_key_points_rw(data['paragraphs'], max_points=12) - if not points: - return JSONResponse({"error": "Không tìm được ý chính"}, status_code=422) - images = data.get('images', []) - slides = [] - for i, point in enumerate(points): - img = images[i] if i < len(images) else (images[-1] if images else '') - if img and 'cdnphoto.dantri' in img: - img = '/api/proxy/img?url=' + _quote2(img, safe='') - slides.append({'text': point, 'image': img, 'index': i + 1}) - summary_text = '\n\n'.join([f"• {s['text']}" for s in slides]) - - # Auto-detect language and emotion - lang, emotion = detect_language_and_emotion(data['title'], summary_text) - # Use preferred voice if provided, otherwise auto-detect - voice = preferred_voice if preferred_voice else get_voice_for_content(data['title'], summary_text) - - post = { - "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)), - "title": data['title'], - "text": summary_text, - "img": images[0] if images else '', - "url": url, - "kind": "slide_summary", - "slides": slides, - "images": images[:10], - "video": "", - "voice": voice, - "emotion": emotion, - "language": lang, - "ts": int(time.time()) - } - posts = _load_wall_posts() - posts.insert(0, post) - _save_wall_posts(posts) - return JSONResponse({"post": post, "slides": slides}) - - -@app.post("/api/rewrite_share") -async def api_rewrite_share(request: Request): - """Rewrite article and post to Tường AI with SLIDES + AI text.""" - body = await request.json() - url = _clean(body.get("url", "")) - ctx = _clean(body.get("context", "")) - preferred_voice = body.get("voice", "") # Accept custom voice selection - if not url and not ctx: - return JSONResponse({"error": "Cần URL hoặc nội dung"}, status_code=400) - data = None - if url and url.startswith("http"): - data = _scrape_article_for_rewrite(url) - if not data and ctx: - paragraphs = [_clean(p) for p in ctx.split('\n') if len(_clean(p)) > 40] - data = {'title': paragraphs[0][:80] if paragraphs else 'Bài viết', 'paragraphs': paragraphs, 'images': [], 'og_img': ''} - if not data or not data.get('paragraphs'): - return JSONResponse({"error": "Không đọc được bài viết"}, status_code=422) - raw_text = '\n'.join(data['paragraphs']) - if len(raw_text) < 50: - raw_text = ctx[:14000] - if len(raw_text) < 50: - return JSONResponse({"error": "Bài viết quá ngắn"}, status_code=422) - domain = '' - try: - from urllib.parse import urlparse - domain = urlparse(url).netloc.replace('www.', '') - except: - pass - - # Generate AI summary text - ai_text = None - try: - import ai_ext - if hasattr(ai_ext, 'qwen_generate'): - prompt = f'Tóm tắt đăng Tường AI:\nTiêu đề: {data["title"]}\n{raw_text[:14000]}\n\n4-6 ý chính. Cuối ghi nguồn.' - ai_text = await ai_ext.qwen_generate(prompt, max_tokens=1000) - except Exception: - pass - if not ai_text or len(ai_text) < 80: - key_pts = _extract_key_points_rw(data['paragraphs'], max_points=12) - if key_pts: - ai_text = '\n\n'.join([f"• {p}" for p in key_pts]) - else: - ai_text = f"Tóm tắt: {data['title']}\n\n{raw_text[:1200]}\n\nNguồn: {domain}" - - # Build slides from key points (FIX: include slides in rewrite_share too!) - points = _extract_key_points_rw(data['paragraphs'], max_points=12) - images = data.get('images', []) - slides = [] - for i, point in enumerate(points): - img = images[i] if i < len(images) else (images[-1] if images else '') - if img and 'cdnphoto.dantri' in img: - img = '/api/proxy/img?url=' + _quote2(img, safe='') - slides.append({'text': point, 'image': img, 'index': i + 1}) - - # Auto-detect language and emotion - lang, emotion = detect_language_and_emotion(data['title'], ai_text) - # Use preferred voice if provided, otherwise auto-detect - voice = preferred_voice if preferred_voice else get_voice_for_content(data['title'], ai_text) - - post = { - "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)), - "title": data['title'], - "text": ai_text, - "img": images[0] if images else '', - "url": url, - "kind": "rewrite", - "slides": slides, - "images": images[:10], - "video": "", - "voice": voice, - "emotion": emotion, - "language": lang, - "ts": int(time.time()) - } - posts = _load_wall_posts() - posts.insert(0, post) - _save_wall_posts(posts) - return JSONResponse({"post": post, "slides": slides}) - - -@app.post("/api/url_wall") -async def api_url_wall(request: Request): - """Submit URL to add to Tường AI.""" - body = await request.json() - url = _clean(body.get("url", "")) - if not url or not url.startswith('http'): - return JSONResponse({"error": "URL không hợp lệ"}, status_code=400) - # Reuse rewrite_share logic - req._body = json.dumps({"url": url}).encode() - return await api_rewrite_share(request) - - -# ===== PERSONAL OPINION POST v2: AI tổng hợp bài viết từ quan điểm + nguồn tin HOT ===== - -# ===== KEYWORD EXTRACTION FROM OPINION ===== -_STOP_WORDS_EX = set(""" -và của các những một được trong với cho tại sau trước khi không người -việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì -này đã để về lại nên cũng rất như vì do nếu sẽ nếu thế nhưng mà vẫn -đang vào ra hơn đây đó nào cả cùng đã từng hãy còn chỉ cũng đều khiến -được đã bị bởi qua những lúc cái gì cô chú bác anh chị em bạn tôi mình -ông bà thầy vì vậy chính phải ấy đấy đâu đó thôi nhé đấy ạ nhỉ -ngày tháng năm giờ phút giây tuần tháng quý -""".strip().split()) - -def _extract_keywords_from_opinion(text, max_keywords=5): - """Extract meaningful keywords from user's opinion for news search.""" - if not text: - return [] - text = text.lower() - text = re.sub(r'https?://\S+', '', text) - text = re.sub(r'[^\w\sÀ-ỹ]', ' ', text) - text = re.sub(r'\s+', ' ', text).strip() - words = [w for w in text.split() if len(w) > 2 and w not in _STOP_WORDS_EX] - word_scores = {} - for w in words: - word_scores[w] = word_scores.get(w, 0) + 1 - sorted_words = sorted(word_scores.items(), key=lambda x: -x[1]) - top_words = [w for w, s in sorted_words[:max_keywords]] - phrases = [] - for i in range(len(words) - 1): - phrase = words[i] + ' ' + words[i + 1] - if len(phrase) > 5: - phrases.append(phrase) - phrase_scores = {} - for p in phrases: - phrase_scores[p] = phrase_scores.get(p, 0) + 1 - sorted_phrases = sorted(phrase_scores.items(), key=lambda x: -x[1]) - top_phrases = [p for p, s in sorted_phrases[:3]] - result = [] - for p in top_phrases: - if p not in result: - result.append(p) - for w in top_words: - if w not in result: - result.append(w) - return result[:max_keywords] - - -@app.post("/api/personal_post/preview") -async def api_personal_post_preview(request: Request): - """Preview personal post: fetch full articles, let AI compose logical article with images.""" - body = await request.json() - opinion = _clean(body.get("opinion", "")) - selected_topics = body.get("selected_topics", []) or [] - selected_sources = body.get("selected_sources", []) or [] - - if not opinion or len(opinion) < 10: - return JSONResponse({"error": "Quan điểm cá nhân quá ngắn (cần ít nhất 10 ký tự)"}, status_code=400) - - # Lấy keywords từ QUAN ĐIỂM CÁ NHÂN để tìm nguồn tin chính xác - keywords = _extract_keywords_from_opinion(opinion, max_keywords=5) - if keywords: - selected_topics = keywords[:3] - else: - # Fallback: hot topics - hot = _get_hot_topics() - selected_topics = [t.get("topic", "") for t in hot[:3] if t.get("topic")] - - # Tìm nguồn tin - all_sources = [] - seen_urls = set() - for topic in selected_topics[:3]: - sources = _search_all(topic, limit=5) - for s in sources: - if s.get("url") and s["url"] not in seen_urls: - seen_urls.add(s["url"]) - all_sources.append(s) - if len(all_sources) >= 6: - break - if len(all_sources) >= 6: - break - - for src in selected_sources: - if src.get("url") and src["url"] not in seen_urls: - all_sources.insert(0, src) - - # Scrape nội dung đầy đủ từng nguồn (paragraphs + images) - source_details = [] - source_images = [] - for src in all_sources[:5]: - url = src.get("url", "") - if not url: - continue - try: - art = _scrape_article_for_rewrite(url) - if art: - src_detail = { - "title": art.get("title", src.get("title", "")), - "url": url, - "via": src.get("via", ""), - "paragraphs": art.get("paragraphs", [])[:8], - "images": art.get("images", [])[:3], - "og_image": art.get("og_img", "") - } - source_details.append(src_detail) - # Collect images for proxy - for img in art.get("images", [])[:2]: - if any(x in img for x in ["cdnphoto.dantri", "vnexpress", "vcdn", "refooty"]): - img = "/api/proxy/img?url=" + _quote2(img, safe="") - source_images.append(img) - except: - pass - if len(source_details) >= 5: - break - - # Tạo title từ opinion - opinion_words = re.findall(r"[A-Za-zÀ-ỹ0-9]+", opinion) - title_words = opinion_words[:8] if len(opinion_words) >= 8 else opinion_words[:4] - title = " ".join([w[0].upper() + w[1:] for w in title_words]) if title_words else "Quan điểm cá nhân" - title = title[:80] - - # AI sinh bài viết hoàn chỉnh - ai_text = None - try: - import ai_ext - if hasattr(ai_ext, 'qwen_generate'): - # Build detailed context from source articles - source_context = "" - for i, sd in enumerate(source_details[:5]): - src_title = sd.get("title", "") - src_via = sd.get("via", "") - src_paras = sd.get("paragraphs", []) - source_context += f"\n=== Nguồn {i+1}: {src_title} ({src_via}) ===\n" - for j, p in enumerate(src_paras[:4]): - source_context += f" - {p[:300]}\n" - - prompt = ( - "QUAN ĐIỂM: " + opinion[:500] + "\nNGUỒN: " + source_context[:1000] + "\n\n" - "=== NGUỒN TIN THAM KHẢO ===\n" + source_context + "\n\n" - "=== YÊU CẦU VIẾT BÀI THEO SLIDE ===\n" - "Viết bài thành 5-6 ĐOẠN VĂN NGẮN, mỗi đoạn là 1 SLIDE.\n" - "\n" - "QUAN TRỌNG NHẤT: MỗI SLIDE PHẢI KẾT HỢP QUAN ĐIỂM CÁ NHÂN + NỘI DUNG NGUỒN TIN, KHÔNG PHẢI CHỈ NÓI VỀ NGUỒN TIN.\n" - "\n" - "SLIDE 1 - MỞ ĐẦU:\n" - "- NHIỆN HỮU QUAN ĐIỂM CÁ NHÂN LÊN ĐẦU\n" - "- Giới thiệu chủ đề, nêu rõ quan điểm của bạn (dựa vào QUAN ĐIỂM CÁ NHÂN ở trên)\n" - "- 2-4 câu hoàn chỉnh\n" - "\n" - "SLIDE 2-3-4-5 - PHÂN TÍCH:\n" - "- Mỗi slide: B�Commencer bằng QUAN ĐIỂM CÁ NHÂN, sau đó dẫn chứng từ 1 nguồn tin\n" - "- Ví dụ: \"Theo quan điểm của tôi, đây là vấn đề cần lưu ý. Theo VnExpress...\"\n" - "- Dẫn chứng từ nguồn (ghi rõ tên báo: Theo VnExpress, Theo Thanh Niên...)\n" - "- 2-4 câu hoàn chỉnh mỗi slide\n" - "\n" - "SLIDE 6 - KẾT LUẬN:\n" - "- Tổng kết quan điểm cá nhân, đưa ra nhận định cuối cùng\n" - "- 2-3 câu hoàn chỉnh\n" - "\n" - "Định dạng đầu ra:\n" - "---SLIDE 1---\n" - "[nội dung đoạn văn slide 1]\n" - "---SLIDE 2---\n" - "[nội dung đoạn văn slide 2]\n" - "...v.v...\n" - "\n" - "QUAN TRỌNG:\n" - "- Mỗi slide là 1 đoạn văn HOÀN CHỈNH, 2-4 câu\n" - "- PHẢI KẾT THÚC BẰNG DẤU CHẤM (.) HOẢN TOÀN\n" - "- Kết hợp QUAN ĐIỂM CÁ NHÂN với NỘI DUNG NGUỒN TIN\n" - "- Không gạch đầu dòng, không bullet points\n" - "- Viết liền mạch tự nhiên, giọng văn báo chí\n" - "- Mỗi slide phải khác nhau, không lặp ý\n" - "- Độ dài: 300-600 từ" - ) - ai_text = None # Không dùng AI, để code tự kết hợp opinion + source - except: - pass - - if not ai_text or len(ai_text) < 100: - # Fallback: build article manually - ai_text = "## " + title + "\n\n" - ai_text += opinion + "\n\n" - for i, sd in enumerate(source_details[:5]): - ai_text += "### " + sd.get("title", f"Nguồn {i+1}") + "\n" - for p in sd.get("paragraphs", [])[:3]: - ai_text += p[:250] + "\n" - ai_text += "*Nguồn: " + sd.get("via", "") + "*\n\n" - ai_text += "\n---\n*Bài viết tổng hợp từ quan điểm cá nhân và các nguồn tin liên quan*" - - # Parse slides từ AI output (format: ---SLIDE N--- content) - slides = [] - if ai_text: - # Try to parse the ---SLIDE--- format - pattern = r'---SLIDE\s*(\d+)---\s*\n(.*?)(?=---SLIDE|\Z)' - matches = re.findall(pattern, ai_text, re.DOTALL) - - if matches: - for idx, (num, content) in enumerate(matches): - # Normalize: ensure complete sentences - text = _ensure_sentence_complete(content) - if len(text) > 40: - img = source_images[idx] if idx < len(source_images) else "" - slides.append({"text": text, "image": img, "index": idx + 1}) - - # If we have parsed slides, ensure minimum 3 - if len(slides) < 3: - # Use parsed slides as base, fill remaining from AI text - used_indices = set() - for s in slides: - used_indices.add(s['index'] - 1) - - # Split remaining AI text into more slides - sentences = re.split(r'(?<=[.!?])\s+', ai_text) - current_chunk = "" - next_idx = len(slides) - - for sent in sentences: - sent = _ensure_sentence_complete(sent) - if len(sent) < 20: - continue - - # Skip if this sentence is already in parsed slides - found = False - for slide in slides: - if sent[:50] in slide['text']: - found = True - break - - if found: - continue - - if current_chunk and len(current_chunk + " " + sent) <= 380: - current_chunk += " " + sent - else: - if len(current_chunk) > 50: - img = source_images[next_idx] if next_idx < len(source_images) else "" - slides.append({"text": current_chunk, "image": img, "index": next_idx + 1}) - current_chunk = sent - next_idx += 1 - - # Add final chunk - if len(current_chunk) > 50 and next_idx < 6: - img = source_images[next_idx] if next_idx < len(source_images) else "" - slides.append({"text": current_chunk, "image": img, "index": next_idx + 1}) - - # Ultimate fallback: create slides from opinion + source - if len(slides) < 2: - slides = [] - # Slide 1: opinion - if opinion and len(opinion) > 20: - slides.append({"text": opinion[:450], "image": source_images[0] if source_images else "", "index": 1}) - - # Slide 2-6: from AI text or sources - if ai_text: - sentences = re.split(r'(?<=[.!?])\s+', ai_text) - for i, sent in enumerate(sentences[:5]): - text = _ensure_sentence_complete(_clean(sent)) - if len(text) > 60: - if len(slides) < 6: - img = source_images[len(slides)] if len(slides) < len(source_images) else "" - slides.append({"text": text, "image": img, "index": len(slides) + 1}) - - # Fill remaining with key points from sources - KẾT HỢP VỚI QUAN ĐIỂM CÁ NHÂN - src_idx = len(slides) - while len(slides) < 4 and src_idx < len(source_details): - paragraphs = source_details[src_idx].get("paragraphs", []) - src_title = source_details[src_idx].get("title", "") - src_via = source_details[src_idx].get("via", "") - for p in paragraphs[:2]: - if len(p) > 60 and len(slides) < 6: - # Kết hợp opinion với nội dung source - combined = f"Theo góc nhìn của tôi, {opinion[:100]}... Theo {src_via}: {p[:250]}" - img = source_images[len(slides)] if len(slides) < len(source_images) else "" - slides.append({"text": _ensure_sentence_complete(combined), "image": img, "index": len(slides) + 1}) - break # Mỗi nguồn 1 slide - src_idx += 1 - - # Final fallback: ensure at least 2-3 slides - while len(slides) < 3: - idx = len(slides) - if idx == 0 and opinion: - slides.append({"text": opinion[:400], "image": "", "index": 1}) - elif ai_text: - slides.append({"text": ai_text[idx*300:(idx+1)*300], "image": "", "index": idx + 1}) - else: - slides.append({"text": f"Nguồn tham khảo {idx + 1}", "image": "", "index": idx + 1}) - - preview = { - "title": title, - "text": ai_text, - "opinion": opinion, - "images": source_images[:10], - "sources": source_details[:5], - "slides": slides[:6] # Max 6 slides - } - - return JSONResponse({"preview": preview}) - - -@app.post("/api/personal_post") -async def api_personal_post(request: Request): - """Create and save personal opinion post.""" - body = await request.json() - opinion = _clean(body.get("opinion", "")) - selected_topics = body.get("selected_topics", []) or [] - selected_sources = body.get("selected_sources", []) or [] - custom_title = body.get("custom_title", "") - custom_slides = body.get("custom_slides", []) - - if not opinion or len(opinion) < 10: - return JSONResponse({"error": "Quan điểm cá nhân quá ngắn (cần ít nhất 10 ký tự)"}, status_code=400) - - if not selected_topics: - # Lấy keywords từ QUAN ĐIỂM CÁ NHÂN để tìm nguồn tin chính xác - keywords = _extract_keywords_from_opinion(opinion, max_keywords=5) - if keywords: - selected_topics = keywords[:3] - else: - hot = _get_hot_topics() - selected_topics = [t.get("topic", "") for t in hot[:3] if t.get("topic")] - - all_sources = [] - seen_urls = set() - for topic in selected_topics[:3]: - sources = _search_all(topic, limit=5) - for s in sources: - if s.get("url") and s["url"] not in seen_urls: - seen_urls.add(s["url"]) - all_sources.append(s) - if len(all_sources) >= 6: - break - if len(all_sources) >= 6: - break - - for src in selected_sources: - if src.get("url") and src["url"] not in seen_urls: - all_sources.insert(0, src) - - source_details = [] - source_images = [] - for src in all_sources[:5]: - url = src.get("url", "") - if not url: - continue - try: - art = _scrape_article_for_rewrite(url) - if art: - src_detail = { - "title": art.get("title", src.get("title", "")), - "url": url, - "via": src.get("via", ""), - "paragraphs": art.get("paragraphs", [])[:6], - "images": art.get("images", [])[:2], - "og_image": art.get("og_img", "") - } - source_details.append(src_detail) - for img in art.get("images", [])[:2]: - if any(x in img for x in ["cdnphoto.dantri", "vnexpress", "vcdn", "refooty"]): - img = "/api/proxy/img?url=" + _quote2(img, safe="") - source_images.append(img) - except: - pass - - # Title - if custom_title: - title = custom_title[:80] - else: - opinion_words = re.findall(r"[A-Za-zÀ-ỹ0-9]+", opinion) - title_words = opinion_words[:8] if len(opinion_words) >= 8 else opinion_words[:4] - title = " ".join([w[0].upper() + w[1:] for w in title_words]) if title_words else "Quan điểm cá nhân" - title = title[:80] - - # AI sinh bài - ai_text = None - try: - import ai_ext - if hasattr(ai_ext, 'qwen_generate'): - source_context = "" - for i, sd in enumerate(source_details[:5]): - src_title = sd.get("title", "") - src_via = sd.get("via", "") - src_paras = sd.get("paragraphs", []) - source_context += f"\nNguồn {i+1}: {src_title} ({src_via})\n" - for j, p in enumerate(src_paras[:3]): - source_context += f" - {p[:300]}\n" - prompt = ( - "QUAN ĐIỂM: " + opinion[:500] + "\nNGUỒN: " + source_context[:1000] + "\n\n" - "=== NGUỒN TIN ===\n" + source_context + "\n\n" - "=== YÊU CẦU VIẾT BÀI THEO SLIDE ===\n" - "Viết bài thành 5-6 ĐOẠN VĂN NGẮN, mỗi đoạn là 1 SLIDE.\n" - "\n" - "SLIDE 1 - MỞ ĐẦU: Giới thiệu chủ đề, nêu quan điểm cá nhân (2-4 câu hoàn chỉnh)\n" - "SLIDE 2-3-4-5 - PHÂN TÍCH: Mỗi slide dùng 1 nguồn tin cụ thể, kết hợp quan điểm cá nhân, ghi rõ nguồn (Theo VnExpress...), 2-4 câu hoàn chỉnh, thành 1 đoạn văn hoàn chỉnh\n" - "SLIDE 6 - KẾT LUẬN: Tổng kết quan điểm, nhận định cuối cùng (2-3 câu hoàn chỉnh)\n" - "\n" - "Định dạng:\n" - "---SLIDE 1---\n[đoạn văn hoàn chỉnh kết thúc bằng dấu chấm]\n---SLIDE 2---\n[đoạn văn hoàn chỉnh kết thúc bằng dấu chấm]\n...\n" - "\n" - "QUAN TRỌNG: Mỗi slide là 1 đoạn văn HOÀN CHỈNH, 2-4 câu, PHẢI KẾT THÚC BẰNG DẤU CHẤM (.). Kết hợp QUAN ĐIỂM + NGUỒN TIN. Không gạch đầu dòng. Viết liền mạch. 300-600 từ." - ) - ai_text = None # Không dùng AI, để code tự kết hợp opinion + source - except: - pass - - if not ai_text or len(ai_text) < 100: - ai_text = "## " + title + "\n\n" + opinion + "\n\n" - for i, sd in enumerate(source_details[:5]): - ai_text += "### " + sd.get("title", "") + "\n" - for p in sd.get("paragraphs", [])[:2]: - ai_text += p[:250] + "\n" - ai_text += "\n---\n*Nguồn: " + sd.get("via", "") + "*\n\n" - - # Tạo slides - if custom_slides and len(custom_slides) > 0: - slides = [] - for i, slide in enumerate(custom_slides): - slides.append({ - "text": slide.get("text", ""), - "image": slide.get("image", ""), - "index": i + 1 - }) - else: - slides = [] - # Parse từ AI output (format: ---SLIDE N---) - if ai_text: - pattern = r'---SLIDE\s*(\d+)---\s*\n(.*?)(?=---SLIDE|\Z)' - matches = re.findall(pattern, ai_text, re.DOTALL) - if matches: - for idx, (num, content) in enumerate(matches): - # Normalize: ensure complete sentences - text = _ensure_sentence_complete(content) - if len(text) > 30: - img = source_images[idx] if idx < len(source_images) else "" - slides.append({"text": text, "image": img, "index": idx + 1}) - - # Fallback: split by paragraphs - if len(slides) < 3: - paragraphs = [p.strip() for p in re.split(r'\n\n+', ai_text) if p.strip()] - slides = [] - para_count = 0 - for p in paragraphs: - p = p.strip() - if p.startswith('#') or p.startswith('---') or p.startswith('*Nguồn'): - continue - # Normalize: ensure complete sentences - p_normalized = _ensure_sentence_complete(p) - if len(p_normalized) > 50: - img = source_images[para_count] if para_count < len(source_images) else "" - slides.append({"text": p_normalized, "image": img, "index": para_count + 1}) - para_count += 1 - if para_count >= 6: - break - - if len(slides) < 2: - slides = [] - # Slide 1: QUAN ĐIỂM CÁ NHÂN (BẮT BUỘC) - slides.append({"text": f"Theo quan điểm cá nhân: {opinion[:400]}", "image": source_images[0] if source_images else "", "index": 1}) - - # Slide 2-6: KẾT HỢP QUAN ĐIỂM + SOURCE - for i in range(min(5, len(source_details))): - if len(slides) >= 6: - break - src = source_details[i] - src_via = src.get("via", "") - src_paras = src.get("paragraphs", []) - - src_text = "" - for p in src_paras[:2]: - p = p.strip()[:280] - if len(p) > 50: - src_text = p - break - - if src_text: - combined = f"Theo góc nhìn cá nhân, {opinion[:60]}. Theo {src_via}: {src_text}" - img = source_images[len(slides)] if len(slides) < len(source_images) else (source_images[-1] if source_images else "") - slides.append({"text": _ensure_sentence_complete(combined), "image": img, "index": len(slides) + 1}) - - lang, emotion = detect_language_and_emotion(title, ai_text) - voice = get_voice_for_content(title, ai_text) - - post = { - "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)), - "title": title, - "text": ai_text, - "img": source_images[0] if source_images else "", - "url": "", - "kind": "personal_opinion", - "slides": slides, - "images": source_images[:10], - "video": "", - "voice": voice, - "emotion": emotion, - "language": lang, - "ts": int(time.time()), - "sources": source_details[:5] - } - - posts = _load_wall_posts() - posts.insert(0, post) - _save_wall_posts(posts) - - return JSONResponse({"post": post, "slides": slides}) - -# ===== END PERSONAL OPINION POST v2 ===== + return JSONResponse({"error": "Post not found"}, status_code=404) def _bg(): time.sleep(15) @@ -2411,415 +874,4 @@ def _bg(): time.sleep(90) threading.Thread(target=_bg,daemon=True).start() -# ===== AUTO SCHEDULER: rewrite AI + short at 7/13/19 VN time ===== -_AUTO_SCHEDULE_TIMES = [(7, '07:00'), (13, '13:00'), (19, '19:00')] -_AUTO_LOG = os.path.join(DATA_DIR, 'auto_rewrite_log.json') - -def _load_auto_log(): - try: - if os.path.exists(_AUTO_LOG): - with open(_AUTO_LOG, 'r') as f: - return json.load(f) - except: pass - return {} - -def _save_auto_log(log): - try: - tmp = _AUTO_LOG + '.tmp' - with open(tmp, 'w') as f: - json.dump(log, f) - os.replace(tmp, _AUTO_LOG) - except: pass - -async def _auto_fetch_short(post_id): - """Try to auto-generate a short for a post.""" - try: - import httpx - async with httpx.AsyncClient(timeout=180) as cl: - r = await cl.post( - f"http://localhost:7860/api/ai/short/{post_id}", - json={"voice":"vi-VN-HoaiMyNeural","emotion":"neutral","speed":1.2}, - headers={"Content-Type":"application/json"} - ) - if r.status_code < 300: - sj = r.json() - if sj.get('video'): - posts = _load_wall_posts() - for p in posts: - if p.get('id') == post_id: - p['video'] = sj['video'] - break - _save_wall_posts(posts) - return True - except: pass - return False - -async def _auto_rewrite_one(topic, slot_label, used_urls=None, post_index=0): - """Rewrite one topic: find articles, summarize, post to wall, trigger short. - used_urls: shared set to avoid duplicate articles across topics. - post_index: 0-based index to create multiple posts per topic (0,1,2 = up to 3 posts).""" - from urllib.parse import quote as _q - # Get MORE items to support 1-3 posts per topic - items = _search_all(topic, limit=12) - # Skip URLs already used by another topic - if used_urls is not None: - filtered = [it for it in items if it.get('url') not in used_urls] - if filtered: - items = filtered - if not items or post_index >= len(items): - return False - - # Get article at post_index (0,1,2 for multiple posts) - item = items[post_index] # post_index allows multiple articles per topic - url = item.get('url', '') - title = item.get('title', topic) - if url and used_urls is not None: - used_urls.add(url) - if not url.startswith('http'): - return False - - data = _scrape_article_for_rewrite(url) - if not data or not data.get('paragraphs'): - return False - - raw_text = '\n'.join(data['paragraphs']) - ai_text = None - - # Try AI generation - try: - import ai_ext - prompt = f"Tóm tắt tin tức (tự động {slot_label}):\nTiêu đề: {data['title']}\n{raw_text[:10000]}\n\n4-6 ý chính dạng bullet. Cuối ghi nguồn." - ai_text = await ai_ext.qwen_generate(prompt, max_tokens=1000) - except: pass - - if not ai_text or len(ai_text) < 80: - pts = data['paragraphs'][:6] - ai_text = '\n\n'.join([f"• {p[:300]}" for p in pts]) - via = item.get('via', '') or urlparse(url).netloc.replace('www.', '') - ai_text += f"\n\nNguồn tham khảo: {via}" - - # Build slides - images = data.get('images', []) - pts = data['paragraphs'][:10] - slides = [] - for i, p in enumerate(pts[:8]): - img = images[i] if i < len(images) else (images[-1] if images else data.get('og_img', '')) - slides.append({'text': p[:300], 'image': img, 'index': i + 1}) - - post_id = str(int(time.time() * 1000)) + str(_random2.randint(100, 999)) - post = { - "id": post_id, "title": data.get('title', title)[:200], - "text": ai_text, "img": images[0] if images else data.get('og_img', ''), - "url": url, "kind": "auto_rewrite", "slides": slides, - "images": images[:10], "video": "", - "voice": "vi-VN-HoaiMyNeural", "emotion": "neutral", - "language": "vietnamese", "ts": int(time.time()), - "auto_scheduled": True, "slot": slot_label, - } - - posts = _load_wall_posts() - posts.insert(0, post) - _save_wall_posts(posts) - - # Trigger short generation async - threading.Thread(target=lambda: asyncio.run(_auto_fetch_short(post_id)), daemon=True).start() - return True - -async def _do_scheduled_run(slot_label): - """Main scheduled run: 1-3 posts from 3 different HOT topics (3-9 total), no duplicates.""" - print(f"[auto] Starting scheduled rewrite for {slot_label}") - - # Get top hot topics, skip duplicates - all_topics = _get_hot_topics() - seen_topics = set() - unique_topics = [] - for t in all_topics: - kw = t.get('topic', '').lower().strip() - if kw and len(kw) > 5 and kw not in seen_topics: - is_dup = False - for s in seen_topics: - # Check if one topic is substring of another - if kw in s or s in kw: - is_dup = True - break - if not is_dup: - seen_topics.add(kw) - unique_topics.append(t) - if len(unique_topics) >= 3: - break - - job_topics = [t['topic'] for t in unique_topics[:3] if t.get('topic')] - if not job_topics: - print(f"[auto] No hot topics found, skipping") - return - - print(f"[auto] Running 3 topics: {job_topics}") - - # Track used URLs to avoid cross-topic duplicates - _used_urls = set() - results = [] - - # Process each topic, create 1-3 posts per topic - for jt in job_topics: - for post_idx in range(3): # Try up to 3 posts per topic - try: - ok = await asyncio.wait_for(_auto_rewrite_one(jt, slot_label, _used_urls, post_idx), timeout=120) - if ok: - results.append((jt, post_idx, True)) - print(f"[auto] Created post {post_idx+1} for '{jt}'") - else: - # No more articles for this topic - break - except Exception as e: - print(f"[auto] Error on '{jt}' post {post_idx}: {e}") - results.append((jt, post_idx, False)) - await asyncio.sleep(1) # Small delay between posts - - # Ensure at least 3 posts total (fallback if needed) - successful_posts = sum(1 for _, _, ok in results if ok) - print(f"[auto] Done {slot_label}: {successful_posts} posts created") - - # Log - from datetime import datetime, timezone, timedelta - VN_TZ_SCHED = timezone(timedelta(hours=7)) - today_str = datetime.now(VN_TZ_SCHED).strftime('%Y-%m-%d') - log = _load_auto_log() - if today_str not in log: log[today_str] = {} - log[today_str][slot_label] = { - 'time': datetime.now(VN_TZ_SCHED).strftime('%H:%M:%S'), - 'count': successful_posts, - 'total': len(job_topics), - } - _save_auto_log(log) - -def _scheduler_loop(): - """Check every 60s; trigger at 7:00, 13:00, 19:00 VN time. - On startup, check for any missed slots today and run them immediately.""" - time.sleep(35) - from datetime import datetime, timezone, timedelta - VN_TZ_SCHED = timezone(timedelta(hours=7)) - - _last_run_date = "" - _last_run_slots = set() - - # On startup: check log for missed slots today - try: - start_now = datetime.now(VN_TZ_SCHED) - today_str = start_now.strftime('%Y-%m-%d') - current_hour = start_now.hour - current_minute = start_now.minute - log = _load_auto_log() - today_log = log.get(today_str, {}) - for h, label in _AUTO_SCHEDULE_TIMES: - # Run if slot is past (either strictly earlier hour, or same hour but window has passed) - should_run = False - if h < current_hour: - should_run = True - elif h == current_hour and current_minute > 10: - should_run = True - if should_run and label not in today_log: - print(f"[auto] Detected missed slot {label} (h={h} < now={current_hour}:{current_minute}), running catch-up now") - _run_scheduled_sync(label) - _last_run_slots.add(label) - except Exception as e: - print(f"[auto] Catch-up check error: {e}") - - while True: - try: - now = datetime.now(VN_TZ_SCHED) - today = now.strftime('%Y-%m-%d') - hour = now.hour - minute = now.minute - - if today != _last_run_date: - _last_run_date = today - _last_run_slots = set() - - slot = None - for h, label in _AUTO_SCHEDULE_TIMES: - if hour == h and 0 <= minute < 5: - slot = label - break - - if slot and slot not in _last_run_slots: - _last_run_slots.add(slot) - _run_scheduled_sync(slot) - except Exception as e: - print(f"[auto] Loop error: {e}") - - time.sleep(60) - -threading.Thread(target=_scheduler_loop, daemon=True, name='auto-rewrite-scheduler').start() - -def _wall_cleanup_once(): - """One-time startup cleanup: sort the persistent wall store newest-first - and remove truly-garbage auto posts (no valid image at ALL, no video, no - slides, no embed URL — i.e. a post that can never be displayed). - - NOTE: This is intentionally conservative. Posts with a valid image or a - valid embed/video are NEVER deleted here — they are only hidden at view - time by _wall_posts_for_view() when the image is a placeholder. This - prevents accidental data loss of FPT Short AI posts whose image is valid - but may temporarily lack a short_thumb.""" - try: - time.sleep(8) - posts = _load_wall_posts() - if not isinstance(posts, list): - return - kept = [] - removed = 0 - removed_ids = [] - needs_sort = False - for p in posts: - if not isinstance(p, dict): - removed += 1 - continue - if _is_auto_post(p): - has_img = _post_has_valid_image(p) - has_content = _post_has_slide_info(p) - vid = (p.get("video") or "").strip() - # Only delete auto posts that are completely unrecoverable: - # no valid image AND no video/slides/embed at all. - if not has_img and not has_content and not vid: - removed += 1 - removed_ids.append(p.get('id', '?')) - continue - kept.append(p) - # Sort if any post lacks proper ordering - prev_ts = None - is_sorted = True - for p in kept: - ts = _created_ts(p) - if prev_ts is not None and ts > prev_ts: - is_sorted = False - break - prev_ts = ts - if removed > 0 or not is_sorted: - kept.sort(key=_created_ts, reverse=True) - _save_wall_posts(kept) - print(f"[wall] startup cleanup: removed {removed} garbage posts ({removed_ids[:10]}), kept {len(kept)}, sorted newest-first") - except Exception as e: - print(f"[wall] startup cleanup error: {e}") - -threading.Thread(target=_wall_cleanup_once, daemon=True, name='wall-cleanup').start() - -@app.get('/api/debug/auto_schedule') -async def debug_auto_schedule(slot: str = '07:00'): - """Manually trigger auto scheduler for debugging.""" - try: - # Check if we can access the data directory - log = _load_auto_log() - topics = _get_hot_topics()[:3] - job_topics = [t['topic'] for t in topics if t.get('topic')] - return JSONResponse({ - "slot": slot, - "log": log, - "hot_topics": job_topics, - "wall_posts_count": len(_load_wall_posts()), - "data_dir_writable": os.access(DATA_DIR, os.W_OK) if os.path.isdir(DATA_DIR) else False, - "data_dir_exists": os.path.isdir(DATA_DIR), - }) - except Exception as e: - return JSONResponse({"error": str(e)}, status_code=500) - -def _run_scheduled_sync(slot): - """Run _do_scheduled_run in a separate event loop (for background thread).""" - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(_do_scheduled_run(slot)) - except Exception as e: - print(f"[auto] Background run error: {e}") - finally: - loop.close() - -@app.get('/api/debug/trigger_auto') -async def debug_trigger_auto(slot: str = '19:00'): - """Trigger _do_scheduled_run in background thread (non-blocking).""" - threading.Thread(target=_run_scheduled_sync, args=(slot,), daemon=True).start() - return JSONResponse({"status": "started", "slot": slot}) - -# ===== SHORTS RSS PROXY ENDPOINT ===== -@app.get("/api/shorts/rss") -def shorts_rss(): - """Get shorts from YouTube RSS feeds server-side""" - import xml.etree.ElementTree as ET - import html as html_lib2 - import re as re2 - - YOUTUBE_CHANNELS = { - "baodantri7941": "UC_x5TKhOgd6GhYvv5z4I3jg", - "baosuckhoedoisongboyte": "UCBsY5fXTQLkF_JnH9kLkL4g", - } - - shorts = [] - seen = set() - - for handle, channel_id in YOUTUBE_CHANNELS.items(): - try: - rss_url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}" - r = req.get(rss_url, headers=HEADERS, timeout=15) - if r.status_code != 200: - continue - - root = ET.fromstring(r.text) - ns = { - 'atom': 'http://www.w3.org/2005/Atom', - 'yt': 'http://www.youtube.com/xml/schemas/2015', - 'media': 'http://search.yahoo.com/mrss/' - } - - for entry in root.findall('atom:entry', ns)[:30]: - title_el = entry.find('atom:title', ns) - title = html_lib2.unescape(title_el.text) if title_el is not None and title_el.text else '' - - link_el = entry.find('atom:link', ns) - link = link_el.get('href', '') if link_el is not None else '' - - vid_el = entry.find('yt:videoId', ns) - vid = vid_el.text if vid_el is not None else '' - - if not vid or vid in seen: - continue - - # Check if it's a short - is_short = '#shorts' in title.lower() or '#short' in title.lower() or '/shorts/' in link - - if not is_short: - desc_el = entry.find('media:description', ns) - if desc_el is not None and desc_el.text: - if '#shorts' in desc_el.text.lower(): - is_short = True - - if not is_short: - continue - - seen.add(vid) - - # Get thumbnail - thumb = f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg" - media_group = entry.find('media:group', ns) - if media_group is not None: - thumb_el = media_group.find('media:thumbnail', ns) - if thumb_el is not None: - thumb = thumb_el.get('url', thumb) - - shorts.append({ - 'id': vid, - 'title': title.replace('#shorts', '').replace('#short', '').strip()[:120], - 'img': thumb, - 'link': f'https://www.youtube.com/shorts/{vid}', - 'channel': handle, - 'source': 'yt' - }) - - if len(shorts) >= 40: - break - - except Exception as e: - print(f"RSS error for {handle}: {e}") - continue - - return {"shorts": shorts, "count": len(shorts)} - -app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static') \ No newline at end of file +app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')