Upload ai_ext.py

ai_ext.py

@@ -96,27 +96,20 @@ def _domain(url: str) -> str:
 
 
 async def qwen_generate(prompt: str, image_url: str = None, max_tokens: int = 1200) -> str:
-    """Generate text using Qwen models via Hugging Face Inference API.
+    """Generate text using Llama/Qwen models via Hugging Face Inference API.
     
-    This function provides a resilient implementation that:
-    1. First tries the SDK-based inference client if available
-    2. Falls back to REST API calls to HF router endpoint
-    3. Returns a fallback summary if all else fails
+    Prioritizes Llama-3.3-70B for better creative/opinion writing.
     """
     token = _hf_token()
     errors = []
     
-    # Try HF router API with multiple models
+    # Try HF router API with multiple models - Llama FIRST for opinion writing
     if token:
         models = [
             os.getenv("QWEN_VL_MODEL", ""),
+            "meta-llama/Llama-3.3-70B-Instruct",  # FIRST - best for opinion/analysis
             "Qwen/Qwen2.5-VL-7B-Instruct",
-            "Qwen/Qwen2.5-VL-3B-Instruct", 
-            "Qwen/Qwen2.5-7B-Instruct",
-            "Qwen/Qwen2.5-3B-Instruct",
-            "Qwen/Qwen2.5-1.5B-Instruct",
             "Qwen/Qwen2.5-72B-Instruct",
-            "meta-llama/Llama-3.3-70B-Instruct",
         ]
         # Deduplicate while preserving order
         seen = set()
@@ -138,12 +131,12 @@ async def qwen_generate(prompt: str, image_url: str = None, max_tokens: int = 12
                 payload = {
                     "model": model,
                     "messages": [
-                        {"role": "system", "content": "Bạn là nhà báo chuyên nghiệp tiếng Việt. Hãy viết bài phân tích dựa trên quan điểm cá nhân được cung cấp, kết hợp với nguồn tin tham khảo. Ưu tiên QUAN ĐIỂM CÁ NHÂN hơn là sao chép nguồn tin."},
+                        {"role": "system", "content": "Bạn là nhà báo phản biện chuyên nghiệp. Luôn viết theo quan điểm cá nhân, phân tích sâu, không sao chép nguyên văn nguồn tin."},
                         {"role": "user", "content": user_content},
                     ],
-                    "max_tokens": min(int(max_tokens or 2500), 3000),
-                    "temperature": 0.85,
-                    "top_p": 0.85,
+                    "max_tokens": min(int(max_tokens or 2000), 2500),
+                    "temperature": 0.75,
+                    "top_p": 0.9,
                 }
                 
                 r = requests.post(
@@ -183,20 +176,18 @@ def _fallback_summary_from_prompt(prompt: str, max_units: int = 6) -> str:
     text = re.sub(r"https?://\S+", "", text)
     text = re.sub(r"\s+", " ", text).strip()
     
-    # Split into sentences - extract ALL valid sentences, not just first few
+    # Split into sentences
     sentences = re.split(r"(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])", text)
     units = []
     for s in sentences:
         s = _clean_text(s)
-        if len(s) >= 30:  # Lower threshold to capture more content
+        if len(s) >= 30:
             units.append(s)
     
     if units:
-        # Take up to max_units valid sentences
         result_units = units[:max_units]
         return "\n".join("• " + u for u in result_units)
     if text:
-        # Fallback: take chunks if no sentence boundaries found
         chunks = []
         for i in range(0, min(len(text), max_units * 300), 280):
             chunk = _clean_text(text[i:i+300])
@@ -206,369 +197,4 @@ def _fallback_summary_from_prompt(prompt: str, max_units: int = 6) -> str:
                     break
         if chunks:
             return "\n".join("• " + c for c in chunks)
-    return "• Không có đủ nội dung để tóm tắt."
-
-
-HF_TOKEN = _hf_token()
-QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
-QWEN_TEXT_MODELS = [m.strip() for m in os.getenv(
-    "QWEN_TEXT_MODELS",
-    "Qwen/Qwen2.5-72B-Instruct,meta-llama/Llama-3.3-70B-Instruct,Qwen/Qwen2.5-7B-Instruct"
-).split(",") if m.strip()]
-_WORKING_MODEL_TEXT = None
-_WORKING_MODEL_VL = None
-DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
-SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts")
-HEADERS = {
-    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
-    "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"
-}
-LAST_QWEN_ERROR = ""
-
-
-# ===== MULTILINGUAL VOICES FOR TTS =====
-# Maps voice IDs to edge-tts voice names (only MultilingualNeural voices)
-MULTILINGUAL_VOICES = {
-    # Vietnamese - Native voices
-    "vi-vn-hoaimyneural": "vi-VN-HoaiMyNeural",
-    "vi-vn-namminhneural": "vi-VN-NamMinhNeural",
-    "hoaimy": "vi-VN-HoaiMyNeural",
-    "namminh": "vi-VN-NamMinhNeural",
-    "vi_female": "vi-VN-HoaiMyNeural",
-    "vi_male": "vi-VN-NamMinhNeural",
-    "nu": "vi-VN-HoaiMyNeural",
-    "male": "vi-VN-NamMinhNeural",
-    "female": "vi-VN-HoaiMyNeural",
-    "mien-nam": "vi-VN-HoaiMyNeural",
-    # English - Multilingual
-    "en-us-andrewmultilingualneural": "en-US-AndrewMultilingualNeural",
-    "en-au-williammultilingualneural": "en-AU-WilliamMultilingualNeural",
-    "en_andrew": "en-US-AndrewMultilingualNeural",
-    "andrew": "en-US-AndrewMultilingualNeural",
-    "en_jenny": "en-US-AndrewMultilingualNeural",
-    "jenny": "en-US-AndrewMultilingualNeural",
-    # Portuguese - Thalita Multilingual ONLY
-    "pt-br-thalitamultilingualneural": "pt-BR-ThalitaMultilingualNeural",
-    "pt_thalita": "pt-BR-ThalitaMultilingualNeural",
-    "thalita": "pt-BR-ThalitaMultilingualNeural",
-    "pt_francisco": "pt-BR-ThalitaMultilingualNeural",
-    "pt": "pt-BR-ThalitaMultilingualNeural",
-    # French - Multilingual
-    "fr-fr-viviennemultilingualneural": "fr-FR-VivienneMultilingualNeural",
-    "fr-fr-remymultilingualneural": "fr-FR-RemyMultilingualNeural",
-    "fr_denise": "fr-FR-VivienneMultilingualNeural",
-    "denise": "fr-FR-VivienneMultilingualNeural",
-    "fr": "fr-FR-VivienneMultilingualNeural",
-    # German - Multilingual
-    "de-de-seraphinamultilingualneural": "de-DE-SeraphinaMultilingualNeural",
-    "de-de-florianmultilingualneural": "de-DE-FlorianMultilingualNeural",
-    "de_katja": "de-DE-SeraphinaMultilingualNeural",
-    "katja": "de-DE-SeraphinaMultilingualNeural",
-    "de": "de-DE-SeraphinaMultilingualNeural",
-    # Korean - Hyunsu Multilingual (NOT SunHee)
-    "ko-kr-hyunsumultilingualneural": "ko-KR-HyunsuMultilingualNeural",
-    "ko_sunhee": "ko-KR-HyunsuMultilingualNeural",
-    "sunhee": "ko-KR-HyunsuMultilingualNeural",
-    "ko": "ko-KR-HyunsuMultilingualNeural",
-    # Italian - Multilingual
-    "it-it-giuseppemultilingualneural": "it-IT-GiuseppeMultilingualNeural",
-    # Spanish (fallback to English multilingual)
-    "es_ela": "en-US-AndrewMultilingualNeural",
-    "ela": "en-US-AndrewMultilingualNeural",
-    "es_carlos": "en-US-AndrewMultilingualNeural",
-    "es": "en-US-AndrewMultilingualNeural",
-    # Japanese (fallback to English multilingual)
-    "ja_nanami": "en-US-AndrewMultilingualNeural",
-    "nanami": "en-US-AndrewMultilingualNeural",
-    "ja": "en-US-AndrewMultilingualNeural",
-    # Chinese (fallback to English multilingual)
-    "zh_xiaochen": "en-US-AndrewMultilingualNeural",
-    "xiaochen": "en-US-AndrewMultilingualNeural",
-    "zh": "en-US-AndrewMultilingualNeural",
-}
-
-
-def _detect_voice_emotion(title, text):
-    """Detect appropriate voice and emotion based on content for multilingual TTS."""
-    content = ((title or "") + " " + (text or "")).lower()
-    
-    # World Cup / Football content - use Andrew multilingual
-    if any(kw in content for kw in ["world cup", "wc 2026", "fifa", "bóng đá", "trận đấu", "bóng bóng", "đội tuyển", "cầu thủ"]):
-        return ("andrew", "excited")
-    
-    # News categories - choose appropriate voice
-    if any(kw in content for kw in ["kinh tế", "tài chính", "thị trường", "economics", "finance"]):
-        return ("jenny", "calm")
-    if any(kw in content for kw in ["thiên tai", "bão", "lũ lụt", "cháy nổ", "tai nạn", "disaster", "accident"]):
-        return ("thalita", "serious")
-    if any(kw in content for kw in ["giải trí", "showbiz", "entertainment", "hài hước"]):
-        return ("ela", "happy")
-    if any(kw in content for kw in ["công nghệ", "tech", "technology", "ai", "trí tuệ nhân tạo"]):
-        return ("katja", "excited")
-    
-    # Default Vietnamese
-    return ("hoaimy", "trung_tinh")
-
-
-def _safe_name(s: str) -> str:
-    """Create safe filename from string."""
-    s = re.sub(r"[^\w\-.]", "_", s)
-    return s[:100] if len(s) > 100 else s
-
-
-def _download_image(url: str, fallback_title: str, out_path: str) -> bool:
-    """Download image from URL to path."""
-    if not url:
-        return False
-    try:
-        r = requests.get(url, headers=HEADERS, timeout=15)
-        if r.status_code == 200:
-            os.makedirs(os.path.dirname(out_path), exist_ok=True)
-            with open(out_path, "wb") as f:
-                f.write(r.content)
-            return True
-    except Exception:
-        pass
-    return False
-
-
-def pollination_image_url(topic: str) -> str:
-    """Generate image URL from Pollinations.ai."""
-    return f"https://image.pollinations.ai/prompt/{quote(topic)}?width=1024&height=768&nologo=true&model=flux"
-
-
-# Use the same wall file as app_v2_entry.py for consistency
-WALL_FILE = os.path.join(DATA_DIR, "wall_posts.json")
-
-def _load_ai_wall():
-    """Load AI wall posts from JSON file (uses wall_posts.json for consistency with app_v2_entry)."""
-    try:
-        if os.path.exists(WALL_FILE):
-            with open(WALL_FILE, "r", encoding="utf-8") as f:
-                return json.load(f)
-    except Exception:
-        pass
-    return []
-
-
-def _save_ai_wall(posts):
-    """Save AI wall posts to JSON file (uses wall_posts.json for consistency with app_v2_entry)."""
-    try:
-        os.makedirs(os.path.dirname(WALL_FILE), exist_ok=True)
-        tmp = WALL_FILE + ".tmp"
-        with open(tmp, "w", encoding="utf-8") as f:
-            json.dump(posts[:100], f, ensure_ascii=False)
-        os.replace(tmp, WALL_FILE)
-    except Exception:
-        pass
-
-
-# Helper functions for wall operations
-def _load_wall_posts():
-    """Alias for _load_ai_wall for consistency with app_v2_entry.py."""
-    return _load_ai_wall()
-
-
-def _save_wall_posts(posts):
-    """Alias for _save_ai_wall for consistency with app_v2_entry.py."""
-    return _save_ai_wall(posts)
-
-
-def make_post(title: str, text: str, img: str, url: str, kind: str, sources=None):
-    """Create a post dict with standard fields."""
-    return {
-        "id": str(int(time.time() * 1000)),
-        "title": title,
-        "text": text,
-        "img": img,
-        "url": url,
-        "kind": kind,
-        "sources": sources or [],
-        "ts": int(time.time())
-    }
-
-
-def _short_script(post) -> str:
-    """Extract clean text for TTS from post."""
-    text = post.get("text", "") or post.get("title", "")
-    text = re.sub(r"^[•\-\*]\s*", "", text, flags=re.M)
-    text = re.sub(r"\s*\n\s*", ". ", text)
-    return _clean_text(text)[:2000]  # Increased from 1000 to 2000 for full content
-
-
-# ===== SCRAPER FUNCTIONS (required by ai_patch.py) =====
-def scrape_any_url(url: str) -> dict:
-    """Scrape any URL and extract article content.
-    
-    Returns dict with: title, summary, text, image, og_image, via (domain)
-    """
-    try:
-        r = requests.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
-        r.encoding = 'utf-8'
-        soup = BeautifulSoup(r.text, 'lxml')
-        
-        # Remove scripts, styles, nav, footer
-        for tag in soup.find_all(['script', 'style', 'nav', 'footer', 'aside', 'form']):
-            tag.decompose()
-        
-        # Extract title
-        h1 = soup.find('h1')
-        ogt = soup.find('meta', property='og:title')
-        title = (h1.get_text(strip=True) if h1 else '') or (ogt.get('content', '') if ogt else url)
-        
-        # Extract OG image
-        ogi = soup.find('meta', property='og:image')
-        og_image = ogi.get('content', '') if ogi else ''
-        
-        # Extract article body
-        block = None
-        for sel in ['article', '.singular-content', '.detail-content', '.fck_detail', '.content-detail', '.knc-content', 'main', '.cms-body', '.article__body']:
-            el = soup.select_one(sel)
-            if el and len(el.find_all('p')) >= 2:
-                block = el
-                break
-        if not block:
-            block = soup.body or soup
-        
-        # Extract text from paragraphs
-        paragraphs = []
-        for el in block.find_all(['p', 'h2', 'h3'], recursive=True):
-            t = _clean_text(el.get_text(strip=True))
-            if t and len(t) > 40:
-                paragraphs.append(t)
-        
-        # Extract images
-        images = []
-        for el in block.find_all(['figure', 'img'], recursive=True):
-            im = el if el.name == 'img' else el.find('img')
-            if im:
-                src = im.get('data-src') or im.get('src') or im.get('data-original') or ''
-                if src and 'base64' not in src:
-                    if src.startswith('//'):
-                        src = 'https:' + src
-                    images.append(src)
-        
-        # Prefer OG image as main image
-        image = og_image or (images[0] if images else '')
-        
-        return {
-            'title': title,
-            'summary': paragraphs[0] if paragraphs else '',
-            'text': '\n'.join(paragraphs),
-            'image': image,
-            'og_image': og_image,
-            'via': _domain(url),
-            'images': images
-        }
-    except Exception as e:
-        return {'title': url, 'summary': '', 'text': '', 'image': '', 'og_image': '', 'via': _domain(url), 'error': str(e)}
-
-
-def web_context(topic: str, limit: int = 5) -> tuple:
-    """Get web context for a topic. Returns (context_text, sources_list)."""
-    sources = []
-    try:
-        # Try Google News RSS
-        rss_url = f"https://news.google.com/rss/search?q={quote_plus(topic)}&hl=vi&gl=VN&ceid=VN:vi"
-        r = requests.get(rss_url, headers=HEADERS, timeout=15)
-        r.encoding = 'utf-8'
-        soup = BeautifulSoup(r.text, 'xml')
-        for it in soup.find_all('item')[:limit]:
-            title = it.find('title').get_text(' ', strip=True) if it.find('title') else ''
-            link = it.find('link').get_text(strip=True) if it.find('link') else ''
-            if title and link:
-                sources.append({'title': title, 'url': link, 'via': _domain(link)})
-    except Exception:
-        pass
-    
-    context = f'Trên mạng có nhiều bài viết về "{topic}". Một số nguồn: ' + ', '.join([s.get('title', '') for s in sources[:3]])
-    return context, sources
-
-
-# ===== SHORT FRAME FUNCTION (required by ai_patch.py) =====
-def _make_short_frame(post, img_path, out_path):
-    """Create a short video frame from post and image.
-    
-    Called by ai_patch.py _make_short_frame_full when Image is available.
-    """
-    if Image is None:
-        # Create a minimal frame without PIL - just return success
-        # The caller should handle this case
-        return False
-    
-    W, H = 1080, 1920
-    bg = Image.new("RGB", (W, H), (14, 14, 14))
-    
-    try:
-        im = Image.open(img_path).convert("RGB")
-        target = (1080, 760)
-        im_ratio = im.width / max(1, im.height)
-        target_ratio = target[0] / target[1]
-        
-        if im_ratio > target_ratio:
-            new_h = target[1]
-            new_w = int(new_h * im_ratio)
-        else:
-            new_w = target[0]
-            new_h = int(new_w / im_ratio)
-        
-        im = im.resize((new_w, new_h))
-        left = (new_w - target[0]) // 2
-        top = (new_h - target[1]) // 2
-        im = im.crop((left, top, left + target[0], top + target[1]))
-        bg.paste(im, (0, 0))
-    except Exception:
-        pass
-    
-    draw = ImageDraw.Draw(bg)
-    
-    try:
-        font_title = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 54)
-        font_body = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 38)
-    except Exception:
-        font_title = font_body = None
-    
-    draw.rectangle((0, 720, W, H), fill=(14, 14, 14))
-    margin = 48
-    maxw = W - margin * 2
-    
-    y = 830
-    for ln in _wrap_text(draw, post.get("title", ""), font_title, maxw, 4):
-        draw.text((margin, y), ln, fill=(255, 255, 255), font=font_title)
-        y += 66
-    
-    y += 18
-    text = post.get("text", "")
-    text = re.sub(r"Nguồn tham khảo:.*", "", text, flags=re.S).strip()
-    body_lines = _wrap_text(draw, text, font_body, maxw, 14)
-    for ln in body_lines:
-        draw.text((margin, y), ln, fill=(220, 220, 220), font=font_body)
-        y += 50
-        if y > 1640:
-            break
-    
-    bg.save(out_path, quality=92)
-    return True
-
-
-def _wrap_text(draw, text, font, max_width, max_lines):
-    """Helper for wrapping text in frames."""
-    words = _clean_text(text).split()
-    lines, cur = [], ""
-    for w in words:
-        test = (cur + " " + w).strip()
-        try:
-            width = draw.textbbox((0, 0), test, font=font)[2]
-        except Exception:
-            width = len(test) * 20
-        if width <= max_width:
-            cur = test
-        else:
-            if cur:
-                lines.append(cur)
-            cur = w
-        if len(lines) >= max_lines:
-            break
-    if cur and len(lines) < max_lines:
-        lines.append(cur)
-    return lines
+    return "• Không có đủ nội dung để tóm tắt."