Upload ai_ext.py

ai_ext.py

@@ -77,46 +77,64 @@ HEADERS = {
 LAST_QWEN_ERROR = ""
 
 # ===== TTS VOICE CONFIG =====
+# Multilingual neural voices grouped by country/language
+# Format: key -> {id, gender, name, country, lang, flag}
 TTS_VOICES = {
+    # === VIETNAM ===
     "hoaimy":     {"id": "vi-VN-HoaiMyNeural",    "gender": "female", "name": "Hoài My",     "country": "Việt Nam", "lang": "vi", "flag": "🇻🇳"},
     "namminh":    {"id": "vi-VN-NamMinhNeural",   "gender": "male",   "name": "Nam Minh",    "country": "Việt Nam", "lang": "vi", "flag": "🇻🇳"},
+    # === USA (English) ===
     "en_us_f":    {"id": "en-US-AvaNeural",        "gender": "female", "name": "Ava",         "country": "Mỹ",       "lang": "en", "flag": "🇺🇸"},
     "en_us_m":    {"id": "en-US-BrianNeural",      "gender": "male",   "name": "Brian",       "country": "Mỹ",       "lang": "en", "flag": "🇺🇸"},
+    # === UK (English) ===
     "en_gb_f":    {"id": "en-GB-SoniaNeural",      "gender": "female", "name": "Sonia",       "country": "Anh",      "lang": "en", "flag": "🇬🇧"},
     "en_gb_m":    {"id": "en-GB-RyanNeural",       "gender": "male",   "name": "Ryan",        "country": "Anh",      "lang": "en", "flag": "🇬🇧"},
+    # === CHINA (Mandarin) ===
     "zh_cn_f":    {"id": "zh-CN-XiaoxiaoNeural",   "gender": "female", "name": "Xiaoxiao",    "country": "Trung Quốc", "lang": "zh", "flag": "🇨🇳"},
     "zh_cn_m":    {"id": "zh-CN-YunyangNeural",    "gender": "male",   "name": "Yunyang",     "country": "Trung Quốc", "lang": "zh", "flag": "🇨🇳"},
     "zh_hk_f":    {"id": "zh-HK-HiuGaaiNeural",    "gender": "female", "name": "HiuGaai",     "country": "Hồng Kông", "lang": "zh", "flag": "🇭🇰"},
+    # === JAPANESE ===
     "ja_jp_f":    {"id": "ja-JP-NanamiNeural",     "gender": "female", "name": "Nanami",      "country": "Nhật Bản", "lang": "ja", "flag": "🇯🇵"},
     "ja_jp_m":    {"id": "ja-JP-KeitaNeural",      "gender": "male",   "name": "Keita",       "country": "Nhật Bản", "lang": "ja", "flag": "🇯🇵"},
+    # === KOREAN ===
     "ko_kr_f":    {"id": "ko-KR-SunHiNeural",      "gender": "female", "name": "Sun-Hi",      "country": "Hàn Quốc", "lang": "ko", "flag": "🇰🇷"},
     "ko_kr_m":    {"id": "ko-KR-InJoonNeural",     "gender": "male",   "name": "InJoon",      "country": "Hàn Quốc", "lang": "ko", "flag": "🇰🇷"},
+    # === FRENCH ===
     "fr_fr_f":    {"id": "fr-FR-DeniseNeural",      "gender": "female", "name": "Denise",      "country": "Pháp",     "lang": "fr", "flag": "🇫🇷"},
     "fr_fr_m":    {"id": "fr-FR-HenriNeural",       "gender": "male",   "name": "Henri",       "country": "Pháp",     "lang": "fr", "flag": "🇫🇷"},
+    # === GERMAN ===
     "de_de_f":    {"id": "de-DE-KatjaNeural",       "gender": "female", "name": "Katja",       "country": "Đức",      "lang": "de", "flag": "🇩🇪"},
     "de_de_m":    {"id": "de-DE-KillianNeural",     "gender": "male",   "name": "Killian",     "country": "Đức",      "lang": "de", "flag": "🇩🇪"},
+    # === SPANISH ===
     "es_es_f":    {"id": "es-ES-XimenaNeural",      "gender": "female", "name": "Ximena",      "country": "Tây Ban Nha", "lang": "es", "flag": "🇪🇸"},
     "es_mx_m":    {"id": "es-MX-JorgeNeural",       "gender": "male",   "name": "Jorge",       "country": "Mexico",   "lang": "es", "flag": "🇲🇽"},
+    # === THAI ===
     "th_th_f":    {"id": "th-TH-PremwadeeNeural",   "gender": "female", "name": "Premwadee",   "country": "Thái Lan", "lang": "th", "flag": "🇹🇭"},
     "th_th_m":    {"id": "th-TH-NiwatNeural",       "gender": "male",   "name": "Niwat",       "country": "Thái Lan", "lang": "th", "flag": "🇹🇭"},
+    # === HINDI ===
     "hi_in_f":    {"id": "hi-IN-SwaraNeural",       "gender": "female", "name": "Swara",       "country": "Ấn Độ",    "lang": "hi", "flag": "🇮🇳"},
     "hi_in_m":    {"id": "hi-IN-MadhurNeural",      "gender": "male",   "name": "Madhur",      "country": "Ấn Độ",    "lang": "hi", "flag": "🇮🇳"},
 }
 TTS_DEFAULT_VOICE = "hoaimy"
-TTS_DEFAULT_SPEED = 1.2
+TTS_DEFAULT_SPEED = 1.2  # 1.2x speed for faster reading
 
+# Topic → voice mapping (auto-detect based on topic keywords)
 TOPIC_VOICE_MAP = {
+    # Sports → male voice
     "bóng đá": "namminh", "thể thao": "namminh", "world cup": "namminh",
     "premier league": "namminh", "champions league": "namminh", "la liga": "namminh",
     "serie a": "namminh", "bundesliga": "namminh", "v-league": "namminh",
     "tennis": "namminh", "olympic": "namminh", "f1": "namminh", "moto": "namminh",
+    # Lifestyle/Health/Entertainment → female voice
     "sức khỏe": "hoaimy", "làm đẹp": "hoaimy", "giải trí": "hoaimy",
     "âm nhạc": "hoaimy", "phim": "hoaimy", "thời trang": "hoaimy",
     "ẩm thực": "hoaimy", "du lịch": "hoaimy", "gia đình": "hoaimy",
     "tình yêu": "hoaimy", "hôn nhân": "hoaimy", "mẹ và bé": "hoaimy",
+    # Tech/Science → male voice
     "công nghệ": "namminh", "ai": "namminh", "robot": "namminh",
     "khoa học": "namminh", "vũ trụ": "namminh", "điện thoại": "namminh",
     "laptop": "namminh", "game": "namminh",
+    # News/Politics/Economy → male voice
     "chính trị": "namminh", "kinh tế": "namminh", "tài chính": "namminh",
     "chứng khoán": "namminh", "ngân hàng": "namminh", "thị trường": "namminh",
     "xã hội": "namminh", "pháp luật": "namminh", "giáo dục": "namminh",
@@ -124,6 +142,7 @@ TOPIC_VOICE_MAP = {
 
 
 def _detect_voice_for_topic(title: str, text: str) -> str:
+    """Auto-detect the best voice based on topic keywords."""
     combined = (title + " " + text[:500]).lower()
     for keyword, voice_id in TOPIC_VOICE_MAP.items():
         if keyword in combined:
@@ -146,109 +165,57 @@ def _safe_name(s):
     return re.sub(r"[^a-zA-Z0-9_-]+", "_", str(s))[:80]
 
 
-# ===== CLEAN AI OUTPUT — COMPLETE REWRITE =====
+# ===== CLEAN AI OUTPUT =====
 def _clean_ai_output(text: str) -> str:
-    """
-    Aggressively clean AI output:
-    1. Remove markdown artifacts
-    2. Remove instruction leakage / meta-commentary
-    3. Split into sentences, deduplicate (exact + near-match)
-    4. Trim at complete sentence boundary (no mid-sentence cuts)
-    5. Remove consecutive duplicate lines
-    """
+    """Remove markdown artifacts, instruction leakage, and aggressively dedup content."""
     if not text:
         return ""
-
-    # --- Step 1: Remove markdown ---
+    # Remove markdown headings, bold, italic, horizontal rules
     text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
     text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
     text = re.sub(r'\*([^*]+)\*', r'\1', text)
     text = re.sub(r'^---+\s*$', '', text, flags=re.MULTILINE)
     text = re.sub(r'^[-*_]{3,}\s*$', '', text, flags=re.MULTILINE)
-
-    # --- Step 2: Remove instruction leakage (entire lines) ---
-    leakage_patterns = [
+    # Remove common AI instruction leakage phrases (entire line)
+    leakage = [
         r'Dưới đây là', r'Theo yêu cầu', r'Tôi sẽ viết', r'Tôi sẽ tóm tắt',
         r'Đây là bài', r'Đây là nội dung', r'Bài viết sau đây',
         r'Nội dung (tóm tắt|chính)', r'Nhiệm vụ', r'Vai trò', r'Tôi là',
         r'Dựa trên.*tôi sẽ', r'Hãy', r'Bạn cần', r'Đọc bài viết',
         r'Tôi xin', r'Xin chào', r'Trân trọng', r'Kính thưa',
         r'Dựa trên.*dưới đây', r'Sau đây là', r'Dưới đây là bài',
-        r'Dựa trên.*tôi đã', r'Theo nội dung', r'Tóm tắt như sau',
-        r'Dưới đây là bài viết', r'Đây là phần', r'Bài tóm tắt',
-        r'Nội dung tóm tắt', r'Đoạn văn sau', r'Viết lại.*sau',
     ]
-    for phrase in leakage_patterns:
-        text = re.sub(r'^[^\n]*' + phrase + r'[^\n]*\n?', '', text, flags=re.MULTILINE | re.IGNORECASE)
-
-    # Remove lines that are just meta-instructions
-    text = re.sub(r'^[^\n]*(?:QUY TẮC|bắt buộc|tối đa|không dùng)[^\n]*\n?', '', text, flags=re.MULTILINE | re.IGNORECASE)
-
+    for phrase in leakage:
+        text = re.sub(r'^' + phrase + r'[^\n]*\n?', '', text, flags=re.MULTILINE | re.IGNORECASE)
     text = re.sub(r'\n{3,}', '\n\n', text)
-    text = text.strip()
-
-    if not text:
-        return ""
-
-    # --- Step 3: Split into sentences & deduplicate ---
-    # Vietnamese sentences end with . ! ? or end of string
-    # We split on sentence-ending punctuation followed by space or end
-    raw_parts = re.split(r'(?<=[.!?])\s+', text.strip())
-
+    # --- Aggressive dedup: split into sentences, remove any that repeat ---
     def _norm(s):
-        """Normalize for comparison: lowercase, collapse whitespace, remove punctuation."""
-        s = re.sub(r'\s+', ' ', s.strip().lower())
-        return re.sub(r'[^\w\s]', '', s)
-
+        return re.sub(r'\s+', ' ', s.strip().lower())
+    raw_parts = re.split(r'(?<=[.!?])\s+', text.strip())
     seen_sentences = set()
     unique_parts = []
-
     for part in raw_parts:
         n = _norm(part)
-        if not n or len(n) < 5:
-            # Very short fragments — keep them if they're not empty
-            if part.strip():
-                unique_parts.append(part)
-            continue
-
-        # Check exact match
-        if n in seen_sentences:
-            continue
-
-        # Check near-duplicate: substring match
         is_dup = False
-        n_clean = re.sub(r'\s+', '', n)
-        for seen in list(seen_sentences):
-            seen_clean = re.sub(r'\s+', '', seen)
-            if not n_clean or not seen_clean:
-                continue
-            # One is substring of the other
-            if n_clean in seen_clean or seen_clean in n_clean:
-                shorter = min(len(n_clean), len(seen_clean))
-                longer = max(len(n_clean), len(seen_clean))
-                if shorter > 15 and shorter / longer > 0.7:
-                    is_dup = True
-                    break
-            # High word overlap
+        if n:
+            if n in seen_sentences:
+                is_dup = True
             else:
-                words_n = set(n.split())
-                words_s = set(seen.split())
-                if len(words_n) >= 3 and len(words_s) >= 3:
-                    overlap = len(words_n & words_s)
-                    ratio = overlap / min(len(words_n), len(words_s))
-                    if ratio > 0.75:
-                        is_dup = True
-                        break
-
+                partial = re.sub(r'\W+', '', n)
+                for seen in seen_sentences:
+                    seen_clean = re.sub(r'\W+', '', seen)
+                    if partial and seen_clean and (partial in seen_clean or seen_clean in partial):
+                        shorter = min(len(partial), len(seen_clean))
+                        longer = max(len(partial), len(seen_clean))
+                        if shorter > 20 and shorter / longer > 0.75:
+                            is_dup = True
+                            break
         if is_dup:
             continue
-
-        seen_sentences.add(n)
+        if n:
+            seen_sentences.add(n)
         unique_parts.append(part)
-
     result = ' '.join(unique_parts).strip()
-
-    # --- Step 4: Remove consecutive duplicate lines ---
     lines = result.split('\n')
     final_lines = []
     prev_line = ""
@@ -259,795 +226,332 @@ def _clean_ai_output(text: str) -> str:
         final_lines.append(line)
         prev_line = stripped
     result = '\n'.join(final_lines).strip()
-
-    # --- Step 5: Trim at complete sentence boundary ---
-    # Find the last sentence-ending punctuation and cut there
-    # This prevents mid-sentence truncation
-    if result:
-        # Find all sentence-ending positions
-        end_matches = list(re.finditer(r'[.!?]\s*$', result))
-        if not end_matches:
-            # No ending punctuation — try to find the last complete sentence
-            last_end = -1
-            for m in re.finditer(r'[.!?](?=\s|$)', result):
-                last_end = m.end()
-            if last_end > 0:
-                result = result[:last_end].strip()
-        # else: already ends with punctuation, good
-
-    # --- Step 6: Final cleanup ---
-    # Remove any trailing incomplete sentence fragments after last punctuation
-    # e.g. "Đây là câu 1. Đây là câu 2. Và một" → cut "Và một"
-    last_period = result.rfind('.')
-    last_exclaim = result.rfind('!')
-    last_question = result.rfind('?')
-    last_end = max(last_period, last_exclaim, last_question)
-    if last_end > 0:
-        after = result[last_end + 1:].strip()
-        # If what follows the last sentence end is a short fragment (< 20 chars, no ending punct), cut it
-        if after and len(after) < 20 and not re.search(r'[.!?]$', after):
-            result = result[:last_end + 1].strip()
-
     return result
 
 
-# ===== EXTRACT ALL IMAGES FROM ARTICLE — ENHANCED =====
+# ===== EXTRACT ALL IMAGES FROM ARTICLE =====
 def _extract_all_images(soup, base_url: str) -> List[Dict]:
     """Extract ALL content images from an article page using multi-strategy approach."""
-    images = []
-    seen_urls = set()
-    skip_patterns = [
-        "avatar", "icon", "logo", "button", "banner-ad", "tracking",
-        "beacon", "pixel", "1x1", "spacer", "emoji", "sprite", "placeholder",
-        "advertisement", "ads", "widget", "sidebar", "footer-logo",
-        "social", "share", "comment-avatar", "user-avatar",
-    ]
-
+    images = []; seen_urls = set()
+    skip_patterns = ["avatar","icon","logo","button","banner-ad","tracking","beacon","pixel","1x1","spacer","emoji","sprite","placeholder","advertisement","ads","widget","sidebar","footer-logo"]
     def _add_image(src: str, alt: str = "", source_tag: str = "img"):
-        if not src or src.startswith("data:"):
-            return
-        # Normalize URL
-        src = src.strip()
-        if src.startswith("//"):
-            src = "https:" + src
-        abs_url = urljoin(base_url, src)
-        if abs_url in seen_urls:
-            return
-        # Skip non-content images by URL pattern
-        if any(p in abs_url.lower() for p in skip_patterns):
-            return
-        # Skip very small images (likely icons)
+        if not src or src.startswith("data:"): return
+        abs_url = urljoin(base_url, src.strip())
+        if abs_url in seen_urls: return
+        if any(p in abs_url.lower() for p in skip_patterns): return
         try:
-            parsed = urlparse(abs_url)
-            path = parsed.path.lower()
-            if any(path.endswith(ext) for ext in ['.svg', '.ico']):
-                return
-            # Skip if URL contains dimension patterns like 16x16, 32x32
-            if re.search(r'/\d{1,3}x\d{1,3}/', path) or re.search(r'-\d{1,3}x\d{1,3}\.', path):
-                return
-        except Exception:
-            pass
-        seen_urls.add(abs_url)
-        images.append({"url": abs_url, "alt": alt, "source": source_tag})
-
-    # Strategy 1: Standard <img> tags with all lazy-load attributes
+            parsed = urlparse(abs_url); path = parsed.path.lower()
+            if any(path.endswith(ext) for ext in ['.svg','.ico','.gif']): return
+        except Exception: pass
+        seen_urls.add(abs_url); images.append({"url":abs_url,"alt":alt,"source":source_tag})
     for img in soup.find_all("img"):
-        src = (img.get("src") or img.get("data-src") or img.get("data-lazy-src") or
-               img.get("data-original") or img.get("data-srcset", "").split(",")[0].strip().split(" ")[0])
-        _add_image(src, alt=img.get("alt", ""), source_tag="img")
-
-    # Strategy 2: srcset on <img>
+        src = (img.get("src") or img.get("data-src") or img.get("data-lazy-src") or img.get("data-original") or img.get("data-srcset","").split(",")[0].strip().split(" ")[0])
+        _add_image(src, alt=img.get("alt",""), source_tag="img")
     for img in soup.find_all("img", srcset=True):
         for part in img["srcset"].split(","):
             part = part.strip()
-            if part:
-                _add_image(part.split(" ")[0], alt=img.get("alt", ""), source_tag="srcset")
-
-    # Strategy 3: <picture> with <source>
+            if part: _add_image(part.split(" ")[0], alt=img.get("alt",""), source_tag="srcset")
     for picture in soup.find_all("picture"):
         for source in picture.find_all("source"):
-            srcset = source.get("srcset", "")
+            srcset = source.get("srcset","")
             for part in srcset.split(","):
                 part = part.strip()
-                if part:
-                    _add_image(part.split(" ")[0], source_tag="picture/srcset")
+                if part: _add_image(part.split(" ")[0], source_tag="picture/srcset")
         fallback_img = picture.find("img")
-        if fallback_img:
-            _add_image(
-                fallback_img.get("src") or fallback_img.get("data-src"),
-                alt=fallback_img.get("alt", ""),
-                source_tag="picture/img"
-            )
-
-    # Strategy 4: WordPress CMS patterns
+        if fallback_img: _add_image(fallback_img.get("src") or fallback_img.get("data-src"), alt=fallback_img.get("alt",""), source_tag="picture/img")
     for img in soup.find_all("img", class_=re.compile(r"wp-image|size-large|size-full|aligncenter")):
-        _add_image(img.get("data-src") or img.get("src"),
-                   alt=img.get("alt", ""), source_tag="wp-image")
-
-    # Strategy 5: Background images in style attributes
+        _add_image(img.get("data-src") or img.get("src"), alt=img.get("alt",""), source_tag="wp-image")
     for tag in soup.find_all(style=re.compile(r"background-image")):
-        for m in re.findall(r'url\(["\']?(.*?)["\']?\)', tag.get("style", "")):
-            _add_image(m, source_tag="background-style")
-
-    # Strategy 6: og:image (featured/hero image)
+        for m in re.findall(r'url\(["\']?(.*?)["\']?\)', tag.get("style","")): _add_image(m, source_tag="background-style")
     og_image = soup.find("meta", property="og:image")
-    if og_image and og_image.get("content"):
-        _add_image(og_image["content"], source_tag="og:image")
-
-    # Strategy 7: twitter:image
-    tw_image = soup.find("meta", attrs={"name": "twitter:image"})
-    if tw_image and tw_image.get("content"):
-        _add_image(tw_image["content"], source_tag="twitter:image")
-
-    # Strategy 8: <figure> with <figcaption>
+    if og_image and og_image.get("content"): _add_image(og_image["content"], source_tag="og:image")
+    tw_image = soup.find("meta", attrs={"name":"twitter:image"})
+    if tw_image and tw_image.get("content"): _add_image(tw_image["content"], source_tag="twitter:image")
     for figure in soup.find_all("figure"):
         img = figure.find("img")
         if img:
             src = img.get("data-src") or img.get("src")
             figcaption = figure.find("figcaption")
-            alt = figcaption.get_text(strip=True) if figcaption else img.get("alt", "")
+            alt = figcaption.get_text(strip=True) if figcaption else img.get("alt","")
             _add_image(src, alt=alt, source_tag="figure")
-
-    # Strategy 9: <a> tags linking to images
     for a in soup.find_all("a", href=True):
         href = a["href"]
-        if any(href.lower().endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".webp"]):
-            _add_image(href, alt=a.get_text(strip=True)[:80], source_tag="link")
-
-    # Strategy 10: VnExpress-specific — fig-parent / fig-image containers
-    for fig_parent in soup.find_all(class_=re.compile(r"fig-parent|fig-image|image|thumb")):
-        img = fig_parent.find("img")
-        if img:
-            src = img.get("data-src") or img.get("src") or img.get("data-original")
-            _add_image(src, alt=img.get("alt", ""), source_tag="fig-parent")
-        # Also check for background-image in fig-parent
-        style = fig_parent.get("style", "")
-        for m in re.findall(r'url\(["\']?(.*?)["\']?\)', style):
-            _add_image(m, source_tag="fig-parent-bg")
-
-    # Strategy 11: Dân Trí / VietnamNet — image in .content-detail, .article-body
-    for container in soup.find_all(class_=re.compile(r"content-detail|article-body|article_content|detail-content|singular-content|fck_detail|knc-content|cms-body|article__body")):
-        for img in container.find_all("img"):
-            src = img.get("data-src") or img.get("src") or img.get("data-original")
-            _add_image(src, alt=img.get("alt", ""), source_tag="content-block-img")
-
-    # Strategy 12: Generic — any <img> inside <article> or <main>
-    for scope in soup.find_all(["article", "main"]):
-        for img in scope.find_all("img"):
-            src = img.get("data-src") or img.get("src") or img.get("data-original")
-            _add_image(src, alt=img.get("alt", ""), source_tag="article-scope")
-
+        if any(href.lower().endswith(ext) for ext in [".jpg",".jpeg",".png",".webp",".gif"]): _add_image(href, alt=a.get_text(strip=True)[:80], source_tag="link")
     return images
 
-
 # ===== JINA READER =====
 def _reader_url(target_url: str) -> str:
     safe = quote(target_url, safe=":/?#[]@!$&'()*+,;=%")
     return "https://r.jina.ai/http://" + safe
-
 def jina_reader_markdown(url: str) -> str:
     jr = _reader_url(url)
-    r = requests.get(jr, headers={"Accept": "text/markdown,text/plain,*/*", "X-Return-Format": "markdown", "User-Agent": "Mozilla/5.0"}, timeout=35)
-    r.raise_for_status()
-    return r.text or ""
-
+    r = requests.get(jr, headers={"Accept":"text/markdown,text/plain,*/*","X-Return-Format":"markdown","User-Agent":"Mozilla/5.0"}, timeout=35)
+    r.raise_for_status(); return r.text or ""
 def _parse_jina_markdown(md: str, url: str):
-    lines = [x.rstrip() for x in (md or "").splitlines()]
-    title = ""; first_image = ""; all_images = []; content_lines = []; in_content = False
+    lines=[x.rstrip() for x in (md or "").splitlines()]; title=""; first_image=""; all_images=[]; content_lines=[]; in_content=False
     for ln in lines:
-        if ln.startswith("Title:") and not title:
-            title = _clean_text(ln.replace("Title:", "", 1)); continue
-        if ln.startswith("URL Source:"):
-            continue
-        if ln.startswith("Markdown Content:"):
-            in_content = True; continue
-        # Extract ALL images from markdown ![alt](url)
-        for mimg in re.finditer(r'!\[[^\]]*\]\((https?://[^)]+)\)', ln):
-            img_url = mimg.group(1)
-            if img_url not in all_images:
-                all_images.append(img_url)
-            if not first_image:
-                first_image = img_url
+        if ln.startswith("Title:") and not title: title=_clean_text(ln.replace("Title:","",1)); continue
+        if ln.startswith("URL Source:"): continue
+        if ln.startswith("Markdown Content:"): in_content=True; continue
+        for mimg in re.finditer(r'!\[[^\]]*\]\((https?://[^)]+)\)',ln):
+            img_url=mimg.group(1)
+            if img_url not in all_images: all_images.append(img_url)
+            if not first_image: first_image=img_url
         if in_content or (title and not ln.startswith("Title:")):
-            if ln.strip():
-                content_lines.append(ln)
-    text = "\n".join(content_lines)
-    text = re.sub(r'!\[[^\]]*\]\([^)]+\)', '', text)
-    paras = []
-    for part in re.split(r'\n{2,}|\n(?=#{1,3}\s)', text):
-        t = _clean_text(re.sub(r'^#{1,6}\s*', '', part))
-        if len(t) >= 40:
-            paras.append(t)
-        if len(paras) >= 35:
-            break
-    if not title and paras:
-        title = paras[0][:90]
-    return {"url": url, "title": title or url, "summary": paras[0] if paras else "",
-            "text": "\n".join(paras), "image": first_image,
-            "images": all_images, "via": "jina"}
-
-
-# ===== WEB SCRAPE (with full image extraction) =====
+            if ln.strip(): content_lines.append(ln)
+    text="\n".join(content_lines); text=re.sub(r'!\[[^\]]*\]\([^)]+\)','',text); paras=[]
+    for part in re.split(r'\n{2,}|\n(?=#{1,3}\s)',text):
+        t=_clean_text(re.sub(r'^#{1,6}\s*','',part))
+        if len(t)>=40: paras.append(t)
+        if len(paras)>=35: break
+    if not title and paras: title=paras[0][:90]
+    return {"url":url,"title":title or url,"summary":paras[0] if paras else "","text":"\n".join(paras),"image":first_image,"images":all_images,"via":"jina"}
+
+# ===== WEB SCRAPE =====
 def _best_content_block(soup):
-    best, best_score = None, 0
-    for el in soup.find_all(["article", "main", "section", "div"]):
-        ps = el.find_all("p")
-        txt = " ".join(p.get_text(" ", strip=True) for p in ps)
-        score = len(ps) * 100 + len(txt)
-        cls = " ".join(el.get("class", []))
-        if any(k in cls.lower() for k in ["content", "article", "detail", "body", "post", "entry"]):
-            score += 800
-        if score > best_score:
-            best, best_score = el, score
+    best,best_score=None,0
+    for el in soup.find_all(["article","main","section","div"]):
+        ps=el.find_all("p"); txt=" ".join(p.get_text(" ",strip=True) for p in ps); score=len(ps)*100+len(txt)
+        cls=" ".join(el.get("class",[]))
+        if any(k in cls.lower() for k in ["content","article","detail","body","post","entry"]): score+=800
+        if score>best_score: best,best_score=el,score
     return best
-
 def scrape_any_url_direct(url: str):
-    r = requests.get(url, headers=HEADERS, timeout=18)
-    if r.status_code in {401, 403, 406, 409, 429, 451, 503}:
-        raise RuntimeError(f"blocked status {r.status_code}")
-    r.encoding = "utf-8"
-    soup = BeautifulSoup(r.text, "lxml")
-    for tag in soup.find_all(["script", "style", "nav", "footer", "aside", "form", "noscript"]):
-        tag.decompose()
-
-    # Title
-    title = soup.find("h1").get_text(" ", strip=True) if soup.find("h1") else ""
+    r=requests.get(url,headers=HEADERS,timeout=18)
+    if r.status_code in {401,403,406,409,429,451,503}: raise RuntimeError(f"blocked status {r.status_code}")
+    r.encoding="utf-8"; soup=BeautifulSoup(r.text,"lxml")
+    for tag in soup.find_all(["script","style","nav","footer","aside","form","noscript"]): tag.decompose()
+    title=soup.find("h1").get_text(" ",strip=True) if soup.find("h1") else ""
     if not title:
-        ogt = soup.find("meta", property="og:title") or soup.find("meta", attrs={"name": "title"})
-        title = ogt.get("content", "") if ogt else (soup.title.get_text(strip=True) if soup.title else "")
-
-    # Summary
-    desc_tag = soup.find("meta", property="og:description") or soup.find("meta", attrs={"name": "description"})
-    summary = desc_tag.get("content", "") if desc_tag else ""
-
-    # Featured image (og:image)
-    img_tag = soup.find("meta", property="og:image") or soup.find("meta", attrs={"name": "twitter:image"})
-    image = img_tag.get("content", "") if img_tag else ""
-    if image and image.startswith("//"):
-        image = "https:" + image
-
-    # Extract ALL images from the article
-    all_images = _extract_all_images(soup, url)
-    image_urls = [img["url"] for img in all_images]
-
-    # Ensure featured image is first
-    if image and image not in image_urls:
-        image_urls.insert(0, image)
-    elif image in image_urls:
-        image_urls.remove(image)
-        image_urls.insert(0, image)
-
-    # Content paragraphs
-    block = _best_content_block(soup) or soup
-    paras, seen_p = [], set()
+        ogt=soup.find("meta",property="og:title") or soup.find("meta",attrs={"name":"title"})
+        title=ogt.get("content","") if ogt else (soup.title.get_text(strip=True) if soup.title else "")
+    desc_tag=soup.find("meta",property="og:description") or soup.find("meta",attrs={"name":"description"})
+    summary=desc_tag.get("content","") if desc_tag else ""
+    img_tag=soup.find("meta",property="og:image") or soup.find("meta",attrs={"name":"twitter:image"})
+    image=img_tag.get("content","") if img_tag else ""
+    if image and image.startswith("//"): image="https:"+image
+    all_images=_extract_all_images(soup,url); image_urls=[img["url"] for img in all_images]
+    if image and image not in image_urls: image_urls.insert(0,image)
+    elif image in image_urls: image_urls.remove(image); image_urls.insert(0,image)
+    block=_best_content_block(soup) or soup; paras,seen_p=[],set()
     for p in block.find_all("p"):
-        t = _clean_text(p.get_text(" ", strip=True))
-        if len(t) >= 40 and t not in seen_p:
-            seen_p.add(t)
-            paras.append(t)
-        if len(paras) >= 35:
-            break
-
-    if not title and paras:
-        title = paras[0][:90]
-
-    return {
-        "url": url, "title": title or url, "summary": paras[0] if paras else "",
-        "text": "\n".join(paras), "image": image_urls[0] if image_urls else "",
-        "images": image_urls, "via": _domain(url)
-    }
-
+        t=_clean_text(p.get_text(" ",strip=True))
+        if len(t)>=40 and t not in seen_p: seen_p.add(t); paras.append(t)
+        if len(paras)>=35: break
+    if not title and paras: title=paras[0][:90]
+    return {"url":url,"title":title or url,"summary":paras[0] if paras else "","text":"\n".join(paras),"image":image_urls[0] if image_urls else "","images":image_urls,"via":_domain(url)}
 def scrape_any_url(url: str):
-    """Try direct scrape first, fall back to Jina Reader.
-    
-    KEY FIX: When falling back to Jina, MERGE images from both sources
-    instead of overwriting. Direct scrape often finds more inline images.
-    """
-    data = scrape_any_url_direct(url)
-    raw_text = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
-    direct_images = data.get("images", [])
-    
-    if len(raw_text) >= 120:
-        # Direct scrape got enough content — but still try Jina for more images
-        # Only use Jina if direct scrape found very few images
-        if len(direct_images) < 3:
-            try:
-                md = jina_reader_markdown(url)
-                if md:
-                    jr = _parse_jina_markdown(md, url)
-                    if jr.get("images"):
-                        # MERGE: add Jina images that aren't already in direct scrape
-                        existing = set(direct_images)
-                        for img_url in jr["images"]:
-                            if img_url not in existing:
-                                direct_images.append(img_url)
-                                existing.add(img_url)
-                        data["images"] = direct_images
-            except Exception:
-                pass
-        return data
-    
-    # Direct scrape didn't get enough content — fall back to Jina
+    data=scrape_any_url_direct(url); raw_text=(data.get("summary","")+"\n"+data.get("text","")).strip()
+    if len(raw_text)>=120: return data
     try:
-        md = jina_reader_markdown(url)
+        md=jina_reader_markdown(url)
         if md:
-            jr = _parse_jina_markdown(md, url)
+            jr=_parse_jina_markdown(md,url)
             if jr.get("text"):
-                # Keep direct scrape's title/images if Jina's are worse
-                if data.get("title") and data["title"] != url:
-                    jr["title"] = data["title"]
-                # MERGE images: direct scrape images + Jina images
-                merged_images = list(direct_images)  # start with direct scrape images
-                existing = set(merged_images)
-                for img_url in jr.get("images", []):
-                    if img_url not in existing:
-                        merged_images.append(img_url)
-                        existing.add(img_url)
-                jr["images"] = merged_images
-                # Use direct scrape's image as first if available
-                if data.get("image"):
-                    jr["image"] = data["image"]
-                jr["via"] = data.get("via", _domain(url)) + " + jina"
-                return jr
-    except Exception:
-        pass
+                if data.get("title") and data["title"]!=url: jr["title"]=data["title"]
+                if data.get("image"): jr["image"]=data["image"]
+                if data.get("images"): jr["images"]=data["images"]
+                jr["via"]=data.get("via",_domain(url))+" + jina"; return jr
+    except Exception: pass
     return data
 
-
-# ===== POLLINATIONS IMAGE =====
 def pollinations_image_url(topic: str) -> str:
-    prompt = "editorial illustration, Vietnamese news, " + topic
-    return "https://image.pollinations.ai/prompt/" + quote(prompt, safe="") + "?width=1024&height=576&nologo=true"
-
-
-# ===== QWEN AI (strict, concise) =====
-async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 700, image_urls: Optional[List[str]] = None):
-    global LAST_QWEN_ERROR, HF_TOKEN
-    HF_TOKEN = _hf_token()
-    if not HF_TOKEN:
-        LAST_QWEN_ERROR = "Không tìm thấy token"
-        return None
-    if not AsyncInferenceClient:
-        LAST_QWEN_ERROR = "Thiếu huggingface_hub"
-        return None
-    errors = []; models = []
-    for m in [QWEN_VL_MODEL, "Qwen/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]:
-        if m and m not in models:
-            models.append(m)
+    prompt="editorial illustration, Vietnamese news, "+topic
+    return "https://image.pollinations.ai/prompt/"+quote(prompt,safe="")+"?width=1024&height=576&nologo=true"
+
+async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 500, image_urls: Optional[List[str]] = None):
+    global LAST_QWEN_ERROR, HF_TOKEN; HF_TOKEN=_hf_token()
+    if not HF_TOKEN: LAST_QWEN_ERROR="Không tìm thấy token"; return None
+    if not AsyncInferenceClient: LAST_QWEN_ERROR="Thiếu huggingface_hub"; return None
+    errors=[]; models=[]
+    for m in [QWEN_VL_MODEL,"Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen2.5-VL-3B-Instruct"]:
+        if m and m not in models: models.append(m)
     for model in models:
         try:
-            client = AsyncInferenceClient(provider="auto", api_key=HF_TOKEN, timeout=90)
-            content = []
-            # Collect all images: image_urls list takes priority, fall back to single image_url
-            all_img_urls = []
-            if image_urls:
-                all_img_urls = image_urls[:6]  # max 6 images to avoid context overflow
-            elif image_url:
-                all_img_urls = [image_url]
+            client=AsyncInferenceClient(provider="auto",api_key=HF_TOKEN,timeout=90); content=[]
+            all_img_urls=[]
+            if image_urls: all_img_urls=image_urls[:6]
+            elif image_url: all_img_urls=[image_url]
             for img_u in all_img_urls:
-                if img_u and img_u.startswith("http"):
-                    content.append({"type": "image_url", "image_url": {"url": img_u}})
-            content.append({"type": "text", "text": prompt})
-            messages = [
-                {"role": "system", "content": (
-                    "Bạn là biên tập viên báo điện tử tiếng Việt. "
-                    "NHIỆM VỤ: Chỉ TÓM TẮT nội dung, KHÔNG viết lại bài đầy đủ. "
-                    "QUY TẮC CỨNG: "
-                    "(1) KHÔNG lặp lại bất kỳ nội dung nào — mỗi ý chỉ xuất hiện ĐÚNG 1 LẦN. "
-                    "(2) Nếu 2 câu diễn đạt cùng 1 ý → bỏ cây thứ 2. "
-                    "(3) KHÔNG dùng Markdown (##, **, ---, *). "
-                    "(4) KHÔNG viết 'Dưới đây là', 'Tôi sẽ', 'Theo yêu cầu', 'Nhiệm vụ', 'Vai trò', 'Đây là bài tóm tắt'. "
-                    "(5) KHÔNG bịa thông tin ngoài nguồn. "
-                    "(6) Chỉ viết ĐOẠN VĂN THUẦN, không bullet points. "
-                    "(7) Tối đa 200 từ. Ngắn gọn, súc tích. "
-                    "(8) KHÔNG sao chép nguyên văn từ bài gốc. Viết lại bằng cách diễn đạt khác hoàn toàn."
-                )},
-                {"role": "user", "content": content}
-            ]
-            resp = await client.chat_completion(model=model, messages=messages, max_tokens=max_tokens, temperature=0.3, top_p=0.8)
-            txt = (resp.choices[0].message.content or "").strip()
-            if txt:
-                LAST_QWEN_ERROR = ""
-                return txt
-        except Exception as e:
-            errors.append(f"{model}: {type(e).__name__}: {str(e)[:220]}")
-    LAST_QWEN_ERROR = " | ".join(errors) or "Qwen không trả nội dung."
-    print("[qwen errors]", LAST_QWEN_ERROR)
-    return None
-
+                if img_u and img_u.startswith("http"): content.append({"type":"image_url","image_url":{"url":img_u}})
+            content.append({"type":"text","text":prompt})
+            messages=[{"role":"system","content":"Bạn là biên tập viên báo điện tử tiếng Việt. NHIỆM VỤ: Chỉ TÓM TẮT nội dung, KHÔNG viết lại bài đầy đủ. QUY TẮC CỨNG: (1) KHÔNG lặp lại bất kỳ nội dung nào — mỗi ý chỉ xuất hiện ĐÚNG 1 LẦN. (2) Nếu 2 câu diễn đạt cùng 1 ý → bỏ cây thứ 2. (3) KHÔNG dùng Markdown (##, **, ---, *). (4) KHÔNG viết 'Dưới đây là', 'Tôi sẽ', 'Theo yêu cầu', 'Nhiệm vụ', 'Vai trò', 'Đây là bài tóm tắt'. (5) KHÔNG bịa thông tin ngoài nguồn. (6) Chỉ viết ĐOẠN VĂN THUẦN, không bullet points. (7) Tối đa 200 từ. Ngắn gọn, súc tích."},{"role":"user","content":content}]
+            resp=await client.chat_completion(model=model,messages=messages,max_tokens=max_tokens,temperature=0.3,top_p=0.8)
+            txt=(resp.choices[0].message.content or "").strip()
+            if txt: LAST_QWEN_ERROR=""; return txt
+        except Exception as e: errors.append(f"{model}: {type(e).__name__}: {str(e)[:220]}")
+    LAST_QWEN_ERROR=" | ".join(errors) or "Qwen không trả nội dung."; print("[qwen errors]",LAST_QWEN_ERROR); return None
 
-# ===== TTS GENERATION =====
 async def _generate_tts_edge(text: str, voice_id: str, speed: float, out_path: str):
-    if edge_tts is None:
-        raise RuntimeError("edge-tts chưa cài đặt")
-    voice = TTS_VOICES.get(voice_id, TTS_VOICES[TTS_DEFAULT_VOICE])["id"]
-    pct = int((speed - 1.0) * 100)
-    rate = f"+{pct}%" if pct >= 0 else f"{pct}%"
-    communicate = edge_tts.Communicate(text, voice, rate=rate)
-    await communicate.save(out_path)
-
+    if edge_tts is None: raise RuntimeError("edge-tts chưa cài đặt")
+    voice=TTS_VOICES.get(voice_id,TTS_VOICES[TTS_DEFAULT_VOICE])["id"]; pct=int((speed-1.0)*100); rate=f"+{pct}%" if pct>=0 else f"{pct}%"
+    communicate=edge_tts.Communicate(text,voice,rate=rate); await communicate.save(out_path)
 def _generate_tts_gtts(text: str, out_path: str):
-    if gTTS is None:
-        raise RuntimeError("gTTS chưa cài đặt")
-    gTTS(text, lang="vi").save(out_path)
-
+    if gTTS is None: raise RuntimeError("gTTS chưa cài đặt")
+    gTTS(text,lang="vi").save(out_path)
 
-# ===== SHORT VIDEO GENERATION =====
-def _download_image(url, fallback_topic, out_path):
+def _download_image(url,fallback_topic,out_path):
     if url:
         try:
-            r = requests.get(url, headers=HEADERS, timeout=15)
-            if r.status_code == 200 and len(r.content) > 1000:
-                with open(out_path, "wb") as f:
-                    f.write(r.content)
+            r=requests.get(url,headers=HEADERS,timeout=15)
+            if r.status_code==200 and len(r.content)>1000:
+                with open(out_path,"wb") as f: f.write(r.content)
                 return out_path
-        except Exception:
-            pass
-    gen = pollinations_image_url(fallback_topic)
+        except Exception: pass
+    gen=pollinations_image_url(fallback_topic)
     try:
-        r = requests.get(gen, headers=HEADERS, timeout=25)
-        if r.status_code == 200 and len(r.content) > 1000:
-            with open(out_path, "wb") as f:
-                f.write(r.content)
+        r=requests.get(gen,headers=HEADERS,timeout=25)
+        if r.status_code==200 and len(r.content)>1000:
+            with open(out_path,"wb") as f: f.write(r.content)
             return out_path
-    except Exception:
-        pass
-    if Image:
-        Image.new("RGB", (1080, 860), (30, 55, 42)).save(out_path)
-        return out_path
+    except Exception: pass
+    if Image: Image.new("RGB",(1080,860),(30,55,42)).save(out_path); return out_path
     raise RuntimeError("Không tạo được ảnh")
-
-def _make_short_frame(post, img_path, out_path):
-    if Image is None:
-        raise RuntimeError("Pillow chưa sẵn sàng")
-    W, H = 1080, 1920
-    bg = Image.new("RGB", (W, H), (14, 14, 14))
+def _make_short_frame(post,img_path,out_path):
+    if Image is None: raise RuntimeError("Pillow chưa sẵn sàng")
+    W,H=1080,1920; bg=Image.new("RGB",(W,H),(14,14,14))
     try:
-        im = Image.open(img_path).convert("RGB")
-        target = (1080, 860)
-        im_ratio = im.width / im.height
-        target_ratio = target[0] / target[1]
-        if im_ratio > target_ratio:
-            new_h = target[1]; new_w = int(new_h * im_ratio)
-        else:
-            new_w = target[0]; new_h = int(new_w / im_ratio)
-        im = im.resize((new_w, new_h))
-        left = (new_w - target[0]) // 2; top = (new_h - target[1]) // 2
-        im = im.crop((left, top, left + target[0], top + target[1]))
-        bg.paste(im, (0, 0))
-    except Exception:
-        pass
-    draw = ImageDraw.Draw(bg)
-    try:
-        font_title = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 52)
-        font_body = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 40)
-        font_label = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 32)
-    except Exception:
-        font_title = font_body = font_label = None
-    draw.rectangle((0, 780, W, H), fill=(14, 14, 14))
-    draw.text((54, 830), "VNEWS · Tường AI", fill=(92, 184, 122), font=font_label)
-    title = post.get("title", "")
-    words = title.split(); lines_t = []; cur = ""
+        im=Image.open(img_path).convert("RGB"); target=(1080,860); im_ratio=im.width/im.height; target_ratio=target[0]/target[1]
+        if im_ratio>target_ratio: new_h=target[1]; new_w=int(new_h*im_ratio)
+        else: new_w=target[0]; new_h=int(new_w/im_ratio)
+        im=im.resize((new_w,new_h)); left=(new_w-target[0])//2; top=(new_h-target[1])//2; im=im.crop((left,top,left+target[0],top+target[1])); bg.paste(im,(0,0))
+    except Exception: pass
+    draw=ImageDraw.Draw(bg)
+    try: font_title=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",52); font_body=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",40); font_label=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",32)
+    except Exception: font_title=font_body=font_label=None
+    draw.rectangle((0,780,W,H),fill=(14,14,14)); draw.text((54,830),"VNEWS · Tường AI",fill=(92,184,122),font=font_label)
+    title=post.get("title",""); words=title.split(); lines_t=[]; cur=""
     for w in words:
-        if len(cur) + len(w) + 1 <= 24:
-            cur = (cur + " " + w).strip()
+        if len(cur)+len(w)+1<=24: cur=(cur+" "+w).strip()
         else:
             if cur: lines_t.append(cur)
-            cur = w
+            cur=w
     if cur: lines_t.append(cur)
-    draw.multiline_text((54, 900), "\n".join(lines_t[:3]), fill=(255, 255, 255), font=font_title, spacing=10)
-    body_text = post.get("text", "")
-    words_b = body_text.split(); lines_b = []; cur_b = ""
+    draw.multiline_text((54,900),"\n".join(lines_t[:3]),fill=(255,255,255),font=font_title,spacing=10)
+    body_text=post.get("text",""); words_b=body_text.split(); lines_b=[]; cur_b=""
     for w in words_b:
-        if len(cur_b) + len(w) + 1 <= 34:
-            cur_b = (cur_b + " " + w).strip()
+        if len(cur_b)+len(w)+1<=34: cur_b=(cur_b+" "+w).strip()
         else:
             if cur_b: lines_b.append(cur_b)
-            cur_b = w
-        if len(lines_b) >= 10:
-            break
-    if cur_b and len(lines_b) < 10: lines_b.append(cur_b)
-    draw.multiline_text((54, 1120), "\n".join(lines_b), fill=(220, 220, 220), font=font_body, spacing=12)
-    bg.save(out_path, quality=92)
-
-def _short_script(post, max_chars=700):
-    txt = _clean_text(post.get("text", ""))
-    if len(txt) > max_chars:
-        # Trim at complete sentence
-        trimmed = txt[:max_chars]
-        last_end = -1
-        for m in re.finditer(r'[.!?](?=\s|$)', trimmed):
-            last_end = m.end()
-        if last_end > 0:
-            txt = trimmed[:last_end]
-        else:
-            txt = trimmed.rsplit(" ", 1)[0] + "."
-    title = _clean_text(post.get("title", ""))
-    return f"{title}. {txt}"
-
-async def _generate_short_video(post, post_id: str, voice_id: str = None, speed: float = None) -> str:
+            cur_b=w
+        if len(lines_b)>=10: break
+    if cur_b and len(lines_b)<10: lines_b.append(cur_b)
+    draw.multiline_text((54,1120),"\n".join(lines_b),fill=(220,220,220),font=font_body,spacing=12); bg.save(out_path,quality=92)
+def _short_script(post,max_chars=700):
+    txt=_clean_text(post.get("text",""))
+    if len(txt)>max_chars: txt=txt[:max_chars].rsplit(" ",1)[0]+"."
+    title=_clean_text(post.get("title","")); return f"{title}. {txt}"
+async def _generate_short_video(post,post_id:str,voice_id:str=None,speed:float=None)->str:
     try:
-        os.makedirs(SHORTS_DIR, exist_ok=True)
-        out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
-        if os.path.exists(out_mp4):
-            return "/api/ai/short-file/" + post_id
-
-        work = os.path.join(SHORTS_DIR, _safe_name(post_id))
-        os.makedirs(work, exist_ok=True)
-        img_path = os.path.join(work, "image.jpg")
-        frame_path = os.path.join(work, "frame.jpg")
-        audio_path = os.path.join(work, "voice.mp3")
-
-        _download_image(post.get("img"), post.get("title", "AI news"), img_path)
-        _make_short_frame(post, img_path, frame_path)
-        script = _short_script(post)
-
-        if voice_id is None:
-            voice_id = _detect_voice_for_topic(post.get("title", ""), post.get("text", ""))
-        if speed is None:
-            speed = TTS_DEFAULT_SPEED
-
-        try:
-            await _generate_tts_edge(script, voice_id, speed, audio_path)
+        os.makedirs(SHORTS_DIR,exist_ok=True); out_mp4=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4")
+        if os.path.exists(out_mp4): return "/api/ai/short-file/"+post_id
+        work=os.path.join(SHORTS_DIR,_safe_name(post_id)); os.makedirs(work,exist_ok=True)
+        img_path=os.path.join(work,"image.jpg"); frame_path=os.path.join(work,"frame.jpg"); audio_path=os.path.join(work,"voice.mp3")
+        _download_image(post.get("img"),post.get("title","AI news"),img_path); _make_short_frame(post,img_path,frame_path); script=_short_script(post)
+        if voice_id is None: voice_id=_detect_voice_for_topic(post.get("title",""),post.get("text",""))
+        if speed is None: speed=TTS_DEFAULT_SPEED
+        try: await _generate_tts_edge(script,voice_id,speed,audio_path)
         except Exception as e:
             print(f"[TTS edge-tts error] {e}, falling back to gTTS")
-            if gTTS:
-                _generate_tts_gtts(script, audio_path)
-            else:
-                return ""
-
-        cmd = ["ffmpeg", "-y", "-loop", "1", "-i", frame_path, "-i", audio_path,
-               "-shortest", "-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
-               "-c:a", "aac", "-b:a", "128k", "-vf", "scale=1080:1920", out_mp4]
-        subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180)
-        return "/api/ai/short-file/" + post_id
-    except Exception as e:
-        print(f"[short video error] {e}")
-        return ""
-
-
-# ===== MAKE POST =====
-def make_post(title, text, image, source_url, kind, sources=None, images=None):
-    return {
-        "id": str(int(time.time() * 1000)) + str(random.randint(100, 999)),
-        "title": title, "text": text, "img": image, "url": source_url,
-        "kind": kind, "sources": sources or [], "video": "",
-        "images": images or [], "ts": int(time.time())
-    }
-
-
-# ===== SHARED PROMPT BUILDER — STRICT REWRITE =====
-def _build_rewrite_prompt(title: str, raw: str, images: List[str] = None) -> str:
-    image_info = ""
+            if gTTS: _generate_tts_gtts(script,audio_path)
+            else: return ""
+        cmd=["ffmpeg","-y","-loop","1","-i",frame_path,"-i",audio_path,"-shortest","-c:v","libx264","-tune","stillimage","-pix_fmt","yuv420p","-c:a","aac","-b:a","128k","-vf","scale=1080:1920",out_mp4]
+        subprocess.run(cmd,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=180); return "/api/ai/short-file/"+post_id
+    except Exception as e: print(f"[short video error] {e}"); return ""
+def make_post(title,text,image,source_url,kind,sources=None,images=None):
+    return {"id":str(int(time.time()*1000))+str(random.randint(100,999)),"title":title,"text":text,"img":image,"url":source_url,"kind":kind,"sources":sources or [],"video":"","images":images or [],"ts":int(time.time())}
+
+def _build_rewrite_prompt(title:str,raw:str,images:List[str]=None)->str:
+    image_info=""
     if images:
-        num = len(images)
-        if num == 1:
-            image_info = "\n\nBài viết có 1 ảnh minh họa. Hãy tham khảo ảnh để hiểu ngữ cảnh (nếu phù hợp)."
-        else:
-            image_info = f"\n\nBài viết có {num} ảnh minh họa. Hãy tham khảo tất cả ảnh để hiểu ngữ cảnh và bổ sung thông tin cho bài viết (nếu phù hợp)."
-
-    return f"""Đọc bài viết dưới đây và viết một bài TÓM TẮT NGẮN trên Tường AI.
-
-⚠️ QUY TẮC CỨNG — VI PHẠM = BỎ QUA:
-1. KHÔNG sao chép nguyên văn bất kỳ câu nào từ bài gốc. Phải diễn đạt LẠI hoàn toàn bằng ngôn ngữ riêng.
-2. KHÔNG lặp lại bất kỳ thông tin nào. Mỗi ý chỉ xuất hiện ĐÚNG 1 LẦN.
-3. Nếu 2 câu diễn đạt cùng 1 ý → chỉ giữ 1 câu, bỏ cây còn lại.
-4. KHÔNG dùng Markdown (##, **, ---, *).
-5. KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu", "Nhiệm vụ", "Vai trò", "Đây là bài tóm tắt", "Sau đây là", "Dựa trên".
-6. Viết thành ĐOẠN VĂN THUẦN, mạch lạc, dễ đọc. Không dùng bullet points.
-7. Giữ sự thật, KHÔNG bịa thông tin.
-8. Tối đa 200 từ. Ngắn gọn, đủ ý.
-9. Mỗi câu phải trọn vẹn, không bị ngắt giữa chừng.{image_info}
-
-Tiêu đề gốc: {title}
-
-Nội dung gốc:
-{raw[:14000]}
-
-Chỉ viết phần tóm tắt, không thêm gì khác:"""
-
-
-def _build_topic_prompt(topic: str, ctx: str) -> str:
-    return f"""Viết bài TÓM TẮT NGẮN GỌN về chủ đề: "{topic}".
-
-⚠️ QUY TẮC CỨNG:
-1. KHÔNG sao chép nguyên văn từ nguồn. Phải diễn đạt LẠI hoàn toàn.
-2. KHÔNG lặp lại bất kỳ nội dung nào. Mỗi thông tin chỉ xuất hiện ĐÚNG 1 LẦN.
-3. Nếu 2 câu nói cùng 1 ý → chỉ giữ 1 câu.
-4. KHÔNG dùng Markdown (##, **, ---, *).
-5. KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu", "Nhiệm vụ", "Vai trò".
-6. Viết thành ĐOẠN VĂN THUẦN, mạch lạc. Không dùng bullet points.
-7. Giữ sự thật, KHÔNG bịa.
-8. Tối đa 200 từ. Ngắn gọn, đủ ý.
-9. Mỗi câu phải trọn vẹn.
-
-Nguồn thực tế:
-{ctx[:12000]}
+        num=len(images)
+        if num==1: image_info="\n\nBài viết có 1 ảnh minh họa. Hãy tham khảo ảnh để hiểu ngữ cảnh (nếu phù hợp)."
+        else: image_info=f"\n\nBài viết có {num} ảnh minh họa. Hãy tham khảo tất cả ảnh để hiểu ngữ cảnh và bổ sung thông tin cho bài viết (nếu phù hợp)."
+    return f"""Tóm tắt bài viết sau thành bài TÓM TẮT đăng Tường AI.\n\nQUY TẮC BẮT BUỘC:\n1. Chỉ viết TÓM TẮT các ý chính. KHÔNG sao chép nguyên văn từ bài gốc.\n2. KHÔNG lặp lại bất kỳ nội dung nào. Mỗi thông tin chỉ xuất hiện ĐÚNG 1 LẦN.\n3. Nếu 2 câu nói cùng 1 ý → chỉ giữ 1 câu, bỏ cây còn lại.\n4. KHÔNG dùng Markdown (##, **, ---, *).\n5. KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu", "Nhiệm vụ", "Vai trò", "Đây là bài tóm tắt".\n6. Viết thành ĐOẠN VĂN THUẦN, mạch lạc, dễ đọc. Không dùng bullet points.\n7. Giữ sự thật, KHÔNG bịa thông tin.\n8. Tối đa 200 từ. Ngắn gọn, đủ ý.{image_info}\n\nTiêu đề gốc: {title}\n\nNội dung gốc:\n{raw[:14000]}"""
+def _build_topic_prompt(topic:str,ctx:str)->str:
+    return f"""Viết bài TÓM TẮT NGẮN GỌN về chủ đề: "{topic}".\n\nQUY TẮC BẮT BUỘC:\n1. Chỉ viết TÓM TẮT các ý chính từ nguồn. KHÔNG sao chép nguyên văn.\n2. KHÔNG lặp lại bất kỳ nội dung nào. Mỗi thông tin chỉ xuất hiện ĐÚNG 1 LẦN.\n3. Nếu 2 câu nói cùng 1 ý → chỉ giữ 1 câu.\n4. KHÔNG dùng Markdown (##, **, ---, *).\n5. KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu", "Nhiệm vụ", "Vai trò".\n6. Viết thành ĐOẠN VĂN THUẦN, mạch lạc. Không dùng bullet points.\n7. Giữ sự thật, KHÔNG bịa.\n8. Tối đa 200 từ. Ngắn gọn, đủ ý.\n\nNguồn thực tế:\n{ctx[:12000]}"""
 
-Chỉ viết phần tóm tắt, không thêm gì khác:"""
-
-
-# ===== WRITE ENDPOINTS =====
 @app.post("/api/rewrite_share")
-async def api_rewrite_share(request: Request):
-    body = await request.json()
-    url = _clean_text(body.get("url", ""))
-    if not url.startswith("http"):
-        return JSONResponse({"error": "missing url"}, status_code=400)
-    try:
-        data = scrape_any_url(url)
-    except Exception as e:
-        return JSONResponse({"error": "Không đọc được bài viết: " + str(e)[:180]}, status_code=422)
-    raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
-    if len(raw) < 60:
-        return JSONResponse({"error": "Bài viết quá ngắn để tóm tắt"}, status_code=422)
-
-    images = data.get("images", [])
-    prompt = _build_rewrite_prompt(data.get("title", ""), raw, images)
-    text = await qwen_generate(prompt, image_urls=images if images else None, max_tokens=700)
-    if not text:
-        return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
-    text = _clean_ai_output(text)
-    post = make_post(data.get("title") or "Bài viết", text,
-                     images[0] if images else data.get("image", ""),
-                     url, "rewrite", images=images)
-
-    # Auto-generate short video
-    video_url = await _generate_short_video(post, post["id"])
-    if video_url:
-        post["video"] = video_url
-
-    posts = _load_wall()
-    posts.insert(0, post)
-    _save_wall(posts)
-    return JSONResponse({"post": post})
-
+async def api_rewrite_share(request:Request):
+    body=await request.json(); url=_clean_text(body.get("url",""))
+    if not url.startswith("http"): return JSONResponse({"error":"missing url"},status_code=400)
+    try: data=scrape_any_url(url)
+    except Exception as e: return JSONResponse({"error":"Không đọc được bài viết: "+str(e)[:180]},status_code=422)
+    raw=(data.get("summary","")+"\n"+data.get("text","")).strip()
+    if len(raw)<60: return JSONResponse({"error":"Bài viết quá ngắn để tóm tắt"},status_code=422)
+    images=data.get("images",[]); prompt=_build_rewrite_prompt(data.get("title",""),raw,images)
+    text=await qwen_generate(prompt,image_urls=images if images else None,max_tokens=500)
+    if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503)
+    text=_clean_ai_output(text); post=make_post(data.get("title") or "Bài viết",text,images[0] if images else data.get("image",""),url,"rewrite",images=images)
+    video_url=await _generate_short_video(post,post["id"])
+    if video_url: post["video"]=video_url
+    posts=_load_wall(); posts.insert(0,post); _save_wall(posts); return JSONResponse({"post":post})
 
 @app.post("/api/url_wall")
-async def api_url_wall(request: Request):
-    body = await request.json()
-    url = _clean_text(body.get("url", ""))
-    if not url.startswith("http"):
-        return JSONResponse({"error": "missing url"}, status_code=400)
-    try:
-        data = scrape_any_url(url)
-    except Exception as e:
-        return JSONResponse({"error": "Không scrape được URL: " + str(e)[:180]}, status_code=422)
-    raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
-    if len(raw) < 60:
-        return JSONResponse({"error": "URL không có đủ nội dung"}, status_code=422)
-
-    images = data.get("images", [])
-    prompt = _build_rewrite_prompt(data.get("title", ""), raw, images)
-    text = await qwen_generate(prompt, image_urls=images if images else None, max_tokens=700)
-    if not text:
-        return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
-    text = _clean_ai_output(text)
-    post = make_post(data.get("title") or "Bài viết", text,
-                     images[0] if images else data.get("image", ""),
-                     url, "url", images=images)
-
-    video_url = await _generate_short_video(post, post["id"])
-    if video_url:
-        post["video"] = video_url
-
-    posts = _load_wall()
-    posts.insert(0, post)
-    _save_wall(posts)
-    return JSONResponse({"post": post})
-
+async def api_url_wall(request:Request):
+    body=await request.json(); url=_clean_text(body.get("url",""))
+    if not url.startswith("http"): return JSONResponse({"error":"missing url"},status_code=400)
+    try: data=scrape_any_url(url)
+    except Exception as e: return JSONResponse({"error":"Không scrape được URL: "+str(e)[:180]},status_code=422)
+    raw=(data.get("summary","")+"\n"+data.get("text","")).strip()
+    if len(raw)<60: return JSONResponse({"error":"URL không có đủ nội dung"},status_code=422)
+    images=data.get("images",[]); prompt=_build_rewrite_prompt(data.get("title",""),raw,images)
+    text=await qwen_generate(prompt,image_urls=images if images else None,max_tokens=500)
+    if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503)
+    text=_clean_ai_output(text); post=make_post(data.get("title") or "Bài viết",text,images[0] if images else data.get("image",""),url,"url",images=images)
+    video_url=await _generate_short_video(post,post["id"])
+    if video_url: post["video"]=video_url
+    posts=_load_wall(); posts.insert(0,post); _save_wall(posts); return JSONResponse({"post":post})
 
 @app.post("/api/topic_post")
-async def api_topic_post(request: Request):
-    body = await request.json()
-    topic = _clean_text(body.get("topic", ""))
-    if not topic:
-        return JSONResponse({"error": "missing topic"}, status_code=400)
+async def api_topic_post(request:Request):
+    body=await request.json(); topic=_clean_text(body.get("topic",""))
+    if not topic: return JSONResponse({"error":"missing topic"},status_code=400)
+    ctx=_web_context(topic)
+    if not ctx: return JSONResponse({"error":"Không lấy được dữ liệu cho chủ đề này"},status_code=422)
+    image=pollinations_image_url(topic); prompt=_build_topic_prompt(topic,ctx)
+    text=await qwen_generate(prompt,image_url=image,max_tokens=500)
+    if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503)
+    text=_clean_ai_output(text); post=make_post(topic,text,image,"","topic")
+    video_url=await _generate_short_video(post,post["id"])
+    if video_url: post["video"]=video_url
+    posts=_load_wall(); posts.insert(0,post); _save_wall(posts); return JSONResponse({"post":post})
 
-    ctx = _web_context(topic)
-    if not ctx:
-        return JSONResponse({"error": "Không lấy được dữ liệu cho chủ đề này"}, status_code=422)
-
-    image = pollinations_image_url(topic)
-    prompt = _build_topic_prompt(topic, ctx)
-    text = await qwen_generate(prompt, image_url=image, max_tokens=700)
-    if not text:
-        return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
-    text = _clean_ai_output(text)
-    post = make_post(topic, text, image, "", "topic")
-
-    video_url = await _generate_short_video(post, post["id"])
-    if video_url:
-        post["video"] = video_url
-
-    posts = _load_wall()
-    posts.insert(0, post)
-    _save_wall(posts)
-    return JSONResponse({"post": post})
-
-
-# ===== WALL ENDPOINTS =====
 @app.get("/api/ai_wall")
-def api_ai_wall():
-    return JSONResponse({"posts": _load_wall()[:80]})
-
+def api_ai_wall(): return JSONResponse({"posts":_load_wall()[:80]})
 @app.get("/api/wall")
-def api_wall():
-    return JSONResponse({"posts": _load_wall()[:80]})
+def api_wall(): return JSONResponse({"posts":_load_wall()[:80]})
 
-
-# ===== SHORT VIDEO ENDPOINT =====
 @app.post("/api/ai/short/{post_id}")
-async def api_ai_short(post_id: str, voice: str = Query(default=None), speed: float = Query(default=None)):
-    posts = _load_wall()
-    post = next((p for p in posts if str(p.get("id")) == str(post_id)), None)
-    if not post:
-        return JSONResponse({"error": "post not found"}, status_code=404)
-
-    os.makedirs(SHORTS_DIR, exist_ok=True)
-    out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
-
+async def api_ai_short(post_id:str,voice:str=Query(default=None),speed:float=Query(default=None)):
+    posts=_load_wall(); post=next((p for p in posts if str(p.get("id"))==str(post_id)),None)
+    if not post: return JSONResponse({"error":"post not found"},status_code=404)
+    os.makedirs(SHORTS_DIR,exist_ok=True); out_mp4=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4")
     if os.path.exists(out_mp4) and voice is None and speed is None:
-        video_url = "/api/ai/short-file/" + post_id
-        for i, p in enumerate(posts):
-            if str(p.get("id")) == str(post_id):
-                posts[i]["video"] = video_url
-                break
-        _save_wall(posts)
-        return JSONResponse({"video": video_url})
-
-    if voice is not None and voice not in TTS_VOICES:
-        return JSONResponse({"error": f"voice không hợp lệ. Chọn: {list(TTS_VOICES.keys())}"}, status_code=400)
-
-    video_url = await _generate_short_video(post, post_id, voice_id=voice, speed=speed)
+        video_url="/api/ai/short-file/"+post_id
+        for i,p in enumerate(posts):
+            if str(p.get("id"))==str(post_id): posts[i]["video"]=video_url; break
+        _save_wall(posts); return JSONResponse({"video":video_url})
+    if voice is not None and voice not in TTS_VOICES: return JSONResponse({"error":f"voice không hợp lệ. Chọn: {list(TTS_VOICES.keys())}"},status_code=400)
+    video_url=await _generate_short_video(post,post_id,voice_id=voice,speed=speed)
     if video_url:
-        for i, p in enumerate(posts):
-            if str(p.get("id")) == str(post_id):
-                posts[i]["video"] = video_url
-                break
-        _save_wall(posts)
-        return JSONResponse({"video": video_url})
-    return JSONResponse({"error": "Không tạo được shorts"}, status_code=500)
-
+        for i,p in enumerate(posts):
+            if str(p.get("id"))==str(post_id): posts[i]["video"]=video_url; break
+        _save_wall(posts); return JSONResponse({"video":video_url})
+    return JSONResponse({"error":"Không tạo được shorts"},status_code=500)
 
 @app.get("/api/ai/short-file/{post_id}")
-def api_ai_short_file(post_id: str):
-    path = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
-    if not os.path.exists(path):
-        return JSONResponse({"error": "not found"}, status_code=404)
-    return FileResponse(path, media_type="video/mp4", filename=f"vnews-ai-{post_id}.mp4")
-
+def api_ai_short_file(post_id:str):
+    path=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4")
+    if not os.path.exists(path): return JSONResponse({"error":"not found"},status_code=404)
+    return FileResponse(path,media_type="video/mp4",filename=f"vnews-ai-{post_id}.mp4")
 
 @app.get("/api/ai/status")
 def api_ai_status():
-    return JSONResponse({
-        "has_token": bool(_hf_token()),
-        "client_imported": AsyncInferenceClient is not None,
-        "model": QWEN_VL_MODEL,
-        "last_error": LAST_QWEN_ERROR,
-        "tts_ready": gTTS is not None or edge_tts is not None,
-        "tts_engine": "edge-tts" if edge_tts else ("gtts" if gTTS else "none"),
-        "tts_voices": {k: v["flag"] + " " + v["name"] for k, v in TTS_VOICES.items()},
-        "tts_voice_count": len(TTS_VOICES),
-        "tts_default_speed": TTS_DEFAULT_SPEED,
-    })
-
+    return JSONResponse({"has_token":bool(_hf_token()),"client_imported":AsyncInferenceClient is not None,"model":QWEN_VL_MODEL,"last_error":LAST_QWEN_ERROR,"tts_ready":gTTS is not None or edge_tts is not None,"tts_engine":"edge-tts" if edge_tts else ("gtts" if gTTS else "none"),"tts_voices":{k:v["flag"]+" "+v["name"] for k,v in TTS_VOICES.items()},"tts_voice_count":len(TTS_VOICES),"tts_default_speed":TTS_DEFAULT_SPEED})
 
 @app.get("/api/ai/voices")
 def api_ai_voices():
-    voices_out = {}
-    for k, v in TTS_VOICES.items():
-        voices_out[k] = {
-            "name": v["name"],
-            "gender": v["gender"],
-            "country": v["country"],
-            "lang": v["lang"],
-            "flag": v["flag"],
-            "label": f"{v['flag']} {v['name']} ({v['gender']})",
-        }
-    return JSONResponse({
-        "voices": voices_out,
-        "default_voice": TTS_DEFAULT_VOICE,
-        "default_speed": TTS_DEFAULT_SPEED,
-        "topic_voice_map": TOPIC_VOICE_MAP,
-    })
+    voices_out={}
+    for k,v in TTS_VOICES.items(): voices_out[k]={"name":v["name"],"gender":v["gender"],"country":v["country"],"lang":v["lang"],"flag":v["flag"],"label":f"{v['flag']} {v['name']} ({v['gender']})"}
+    return JSONResponse({"voices":voices_out,"default_voice":TTS_DEFAULT_VOICE,"default_speed":TTS_DEFAULT_SPEED,"topic_voice_map":TOPIC_VOICE_MAP})