ai_ext.py

"""VNEWS AI Extension - rewrite + auto short video generation.
Imported by app_v2_entry.py to register /api/rewrite_share, /api/topic_post,
/api/ai_wall, /api/wall, /api/ai/short endpoints on the main FastAPI app.

Uses main.py's WALL_FILE (wall_posts.json) for unified data store.
TTS: edge-tts (HoaiMy female, NamMinh male) with speed control + gTTS fallback.
"""
import os, re, json, time, random, html as html_lib, subprocess, asyncio
from urllib.parse import quote_plus, quote, urlparse, urljoin
from typing import Optional, List, Dict
import requests
from bs4 import BeautifulSoup
from fastapi import Request, Query
from fastapi.responses import HTMLResponse, JSONResponse, FileResponse

# Try to import main app, but don't fail if it doesn't exist
try:
    from main import app
except ImportError:
    # Create a minimal FastAPI app for standalone testing
    try:
        from fastapi import FastAPI
        app = FastAPI()
    except Exception:
        app = None

# Import wall store from main.py so we read/write the SAME file
try:
    from main import _load_wall, _save_wall, _web_context  # noqa: F401
except ImportError:
    _data_dir = "/data" if os.path.isdir("/data") else "/app/data"
    _wall_file = os.path.join(_data_dir, "wall_posts.json")
    def _load_wall():
        try:
            if os.path.exists(_wall_file):
                with open(_wall_file, "r", encoding="utf-8") as f:
                    return json.load(f)
        except Exception:
            pass
        return []
    def _save_wall(posts):
        try:
            os.makedirs(os.path.dirname(_wall_file), exist_ok=True)
            tmp = _wall_file + ".tmp"
            with open(tmp, "w", encoding="utf-8") as f:
                json.dump(posts[:100], f, ensure_ascii=False)
            os.replace(tmp, _wall_file)
        except Exception:
            pass
    def _web_context(topic):
        return ""

# ai_ext alias for backward compatibility
_load_ai_wall = _load_wall
_save_ai_wall = _save_wall

try:
    from huggingface_hub import AsyncInferenceClient
except Exception:
    AsyncInferenceClient = None
try:
    from gtts import gTTS
except Exception:
    gTTS = None
try:
    from PIL import Image, ImageDraw, ImageFont
except Exception:
    Image = ImageDraw = ImageFont = None
try:
    import edge_tts
except Exception:
    edge_tts = None


def _hf_token():
    for k in ("HF_TOKEN", "HUGGINGFACE_HUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_API_TOKEN"):
        v = os.getenv(k, "").strip()
        if v:
            return v
    return ""


def _clean_text(s: str) -> str:
    """Clean text for processing."""
    s = html_lib.unescape(s or "")
    s = re.sub(r"\s+", " ", s)
    return s.strip()


def _domain(url: str) -> str:
    """Extract domain from URL."""
    try:
        return urlparse(url or "").netloc.replace("www.", "")
    except Exception:
        return ""


async def qwen_generate(prompt: str, image_url: str = None, max_tokens: int = 1200) -> str:
    """Generate text using Qwen models via Hugging Face Inference API.
    
    This function provides a resilient implementation that:
    1. First tries the SDK-based inference client if available
    2. Falls back to REST API calls to HF router endpoint
    3. Returns a fallback summary if all else fails
    """
    token = _hf_token()
    errors = []
    
    # Try HF router API with multiple models
    if token:
        models = [
            os.getenv("QWEN_VL_MODEL", ""),
            "Qwen/Qwen2.5-VL-7B-Instruct",
            "Qwen/Qwen2.5-VL-3B-Instruct", 
            "Qwen/Qwen2.5-7B-Instruct",
            "Qwen/Qwen2.5-3B-Instruct",
            "Qwen/Qwen2.5-1.5B-Instruct",
            "Qwen/Qwen2.5-72B-Instruct",
            "meta-llama/Llama-3.3-70B-Instruct",
        ]
        # Deduplicate while preserving order
        seen = set()
        models = [m for m in models if m and m not in seen and not seen.add(m)]
        
        headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
        
        for model in models:
            try:
                is_vl = "VL" in model and image_url
                if is_vl:
                    user_content = [
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {"type": "text", "text": prompt}
                    ]
                else:
                    user_content = prompt
                
                payload = {
                    "model": model,
                    "messages": [
                        {"role": "system", "content": "Bạn là trợ lý AI tiếng Việt. Trả lời tự nhiên, ngắn gọn, chính xác."},
                        {"role": "user", "content": user_content},
                    ],
                    "max_tokens": min(int(max_tokens or 900), 1400),
                    "temperature": 0.35,
                    "top_p": 0.85,
                }
                
                r = requests.post(
                    "https://router.huggingface.co/v1/chat/completions",
                    headers=headers,
                    json=payload,
                    timeout=95
                )
                
                if r.status_code >= 300:
                    errors.append(f"{model}: HTTP {r.status_code}")
                    continue
                    
                j = r.json()
                txt = (j.get("choices", [{}])[0].get("message", {}).get("content") or "").strip()
                
                if txt:
                    return txt
                    
                errors.append(f"{model}: empty response")
                
            except Exception as e:
                errors.append(f"{model}: {type(e).__name__}")
    
    # Fallback: extractive summary from prompt
    LAST_QWEN_ERROR = errors[-3:] if errors else "unknown error"
    return _fallback_summary_from_prompt(prompt, max_units=6)


def _fallback_summary_from_prompt(prompt: str, max_units: int = 6) -> str:
    """Generate a simple fallback summary when AI is unavailable."""
    text = prompt or ""
    for marker in ["Nội dung nguồn:", "Nội dung bài:", "Nội dung gốc:", "Nội dung:", "Nguồn/bối cảnh internet:"]:
        if marker in text:
            text = text.split(marker, 1)[1]
            break
    text = re.sub(r"https?://\S+", "", text)
    text = re.sub(r"\s+", " ", text).strip()
    
    # Split into sentences - extract ALL valid sentences, not just first few
    sentences = re.split(r"(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])", text)
    units = []
    for s in sentences:
        s = _clean_text(s)
        if len(s) >= 30:  # Lower threshold to capture more content
            units.append(s)
    
    if units:
        # Take up to max_units valid sentences
        result_units = units[:max_units]
        return "\n".join("• " + u for u in result_units)
    if text:
        # Fallback: take chunks if no sentence boundaries found
        chunks = []
        for i in range(0, min(len(text), max_units * 300), 280):
            chunk = _clean_text(text[i:i+300])
            if chunk and chunk not in chunks:
                chunks.append(chunk)
                if len(chunks) >= max_units:
                    break
        if chunks:
            return "\n".join("• " + c for c in chunks)
    return "• Không có đủ nội dung để tóm tắt."


HF_TOKEN = _hf_token()
QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
QWEN_TEXT_MODELS = [m.strip() for m in os.getenv(
    "QWEN_TEXT_MODELS",
    "Qwen/Qwen2.5-72B-Instruct,meta-llama/Llama-3.3-70B-Instruct,Qwen/Qwen2.5-7B-Instruct"
).split(",") if m.strip()]
_WORKING_MODEL_TEXT = None
_WORKING_MODEL_VL = None
DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"
}
LAST_QWEN_ERROR = ""


# ===== MULTILINGUAL VOICES FOR TTS =====
# Maps voice IDs to edge-tts voice names (only MultilingualNeural voices)
MULTILINGUAL_VOICES = {
    # Vietnamese - Native voices
    "vi-vn-hoaimyneural": "vi-VN-HoaiMyNeural",
    "vi-vn-namminhneural": "vi-VN-NamMinhNeural",
    "hoaimy": "vi-VN-HoaiMyNeural",
    "namminh": "vi-VN-NamMinhNeural",
    "vi_female": "vi-VN-HoaiMyNeural",
    "vi_male": "vi-VN-NamMinhNeural",
    "nu": "vi-VN-HoaiMyNeural",
    "male": "vi-VN-NamMinhNeural",
    "female": "vi-VN-HoaiMyNeural",
    "mien-nam": "vi-VN-HoaiMyNeural",
    # English - Multilingual
    "en-us-andrewmultilingualneural": "en-US-AndrewMultilingualNeural",
    "en-au-williammultilingualneural": "en-AU-WilliamMultilingualNeural",
    "en_andrew": "en-US-AndrewMultilingualNeural",
    "andrew": "en-US-AndrewMultilingualNeural",
    "en_jenny": "en-US-AndrewMultilingualNeural",
    "jenny": "en-US-AndrewMultilingualNeural",
    # Portuguese - Thalita Multilingual ONLY
    "pt-br-thalitamultilingualneural": "pt-BR-ThalitaMultilingualNeural",
    "pt_thalita": "pt-BR-ThalitaMultilingualNeural",
    "thalita": "pt-BR-ThalitaMultilingualNeural",
    "pt_francisco": "pt-BR-ThalitaMultilingualNeural",
    "pt": "pt-BR-ThalitaMultilingualNeural",
    # French - Multilingual
    "fr-fr-viviennemultilingualneural": "fr-FR-VivienneMultilingualNeural",
    "fr-fr-remymultilingualneural": "fr-FR-RemyMultilingualNeural",
    "fr_denise": "fr-FR-VivienneMultilingualNeural",
    "denise": "fr-FR-VivienneMultilingualNeural",
    "fr": "fr-FR-VivienneMultilingualNeural",
    # German - Multilingual
    "de-de-seraphinamultilingualneural": "de-DE-SeraphinaMultilingualNeural",
    "de-de-florianmultilingualneural": "de-DE-FlorianMultilingualNeural",
    "de_katja": "de-DE-SeraphinaMultilingualNeural",
    "katja": "de-DE-SeraphinaMultilingualNeural",
    "de": "de-DE-SeraphinaMultilingualNeural",
    # Korean - Hyunsu Multilingual (NOT SunHee)
    "ko-kr-hyunsumultilingualneural": "ko-KR-HyunsuMultilingualNeural",
    "ko_sunhee": "ko-KR-HyunsuMultilingualNeural",
    "sunhee": "ko-KR-HyunsuMultilingualNeural",
    "ko": "ko-KR-HyunsuMultilingualNeural",
    # Italian - Multilingual
    "it-it-giuseppemultilingualneural": "it-IT-GiuseppeMultilingualNeural",
    # Spanish (fallback to English multilingual)
    "es_ela": "en-US-AndrewMultilingualNeural",
    "ela": "en-US-AndrewMultilingualNeural",
    "es_carlos": "en-US-AndrewMultilingualNeural",
    "es": "en-US-AndrewMultilingualNeural",
    # Japanese (fallback to English multilingual)
    "ja_nanami": "en-US-AndrewMultilingualNeural",
    "nanami": "en-US-AndrewMultilingualNeural",
    "ja": "en-US-AndrewMultilingualNeural",
    # Chinese (fallback to English multilingual)
    "zh_xiaochen": "en-US-AndrewMultilingualNeural",
    "xiaochen": "en-US-AndrewMultilingualNeural",
    "zh": "en-US-AndrewMultilingualNeural",
}


def _detect_voice_emotion(title, text):
    """Detect appropriate voice and emotion based on content for multilingual TTS."""
    content = ((title or "") + " " + (text or "")).lower()
    
    # World Cup / Football content - use Andrew multilingual
    if any(kw in content for kw in ["world cup", "wc 2026", "fifa", "bóng đá", "trận đấu", "bóng bóng", "đội tuyển", "cầu thủ"]):
        return ("andrew", "excited")
    
    # News categories - choose appropriate voice
    if any(kw in content for kw in ["kinh tế", "tài chính", "thị trường", "economics", "finance"]):
        return ("jenny", "calm")
    if any(kw in content for kw in ["thiên tai", "bão", "lũ lụt", "cháy nổ", "tai nạn", "disaster", "accident"]):
        return ("thalita", "serious")
    if any(kw in content for kw in ["giải trí", "showbiz", "entertainment", "hài hước"]):
        return ("ela", "happy")
    if any(kw in content for kw in ["công nghệ", "tech", "technology", "ai", "trí tuệ nhân tạo"]):
        return ("katja", "excited")
    
    # Default Vietnamese
    return ("hoaimy", "trung_tinh")


def _safe_name(s: str) -> str:
    """Create safe filename from string."""
    s = re.sub(r"[^\w\-.]", "_", s)
    return s[:100] if len(s) > 100 else s


def _download_image(url: str, fallback_title: str, out_path: str) -> bool:
    """Download image from URL to path."""
    if not url:
        return False
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        if r.status_code == 200:
            os.makedirs(os.path.dirname(out_path), exist_ok=True)
            with open(out_path, "wb") as f:
                f.write(r.content)
            return True
    except Exception:
        pass
    return False


def pollination_image_url(topic: str) -> str:
    """Generate image URL from Pollinations.ai."""
    return f"https://image.pollinations.ai/prompt/{quote(topic)}?width=1024&height=768&nologo=true&model=flux"


# Use the same wall file as app_v2_entry.py for consistency
WALL_FILE = os.path.join(DATA_DIR, "wall_posts.json")

def _load_ai_wall():
    """Load AI wall posts from JSON file (uses wall_posts.json for consistency with app_v2_entry)."""
    try:
        if os.path.exists(WALL_FILE):
            with open(WALL_FILE, "r", encoding="utf-8") as f:
                return json.load(f)
    except Exception:
        pass
    return []


def _save_ai_wall(posts):
    """Save AI wall posts to JSON file (uses wall_posts.json for consistency with app_v2_entry)."""
    try:
        os.makedirs(os.path.dirname(WALL_FILE), exist_ok=True)
        tmp = WALL_FILE + ".tmp"
        with open(tmp, "w", encoding="utf-8") as f:
            json.dump(posts[:100], f, ensure_ascii=False)
        os.replace(tmp, WALL_FILE)
    except Exception:
        pass


# Helper functions for wall operations
def _load_wall_posts():
    """Alias for _load_ai_wall for consistency with app_v2_entry.py."""
    return _load_ai_wall()


def _save_wall_posts(posts):
    """Alias for _save_ai_wall for consistency with app_v2_entry.py."""
    return _save_ai_wall(posts)


def make_post(title: str, text: str, img: str, url: str, kind: str, sources=None):
    """Create a post dict with standard fields."""
    return {
        "id": str(int(time.time() * 1000)),
        "title": title,
        "text": text,
        "img": img,
        "url": url,
        "kind": kind,
        "sources": sources or [],
        "ts": int(time.time())
    }


def _short_script(post) -> str:
    """Extract clean text for TTS from post."""
    text = post.get("text", "") or post.get("title", "")
    text = re.sub(r"^[•\-\*]\s*", "", text, flags=re.M)
    text = re.sub(r"\s*\n\s*", ". ", text)
    return _clean_text(text)[:2000]  # Increased from 1000 to 2000 for full content


# ===== SCRAPER FUNCTIONS (required by ai_patch.py) =====
def scrape_any_url(url: str) -> dict:
    """Scrape any URL and extract article content.
    
    Returns dict with: title, summary, text, image, og_image, via (domain)
    """
    try:
        r = requests.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text, 'lxml')
        
        # Remove scripts, styles, nav, footer
        for tag in soup.find_all(['script', 'style', 'nav', 'footer', 'aside', 'form']):
            tag.decompose()
        
        # Extract title
        h1 = soup.find('h1')
        ogt = soup.find('meta', property='og:title')
        title = (h1.get_text(strip=True) if h1 else '') or (ogt.get('content', '') if ogt else url)
        
        # Extract OG image
        ogi = soup.find('meta', property='og:image')
        og_image = ogi.get('content', '') if ogi else ''
        
        # Extract article body
        block = None
        for sel in ['article', '.singular-content', '.detail-content', '.fck_detail', '.content-detail', '.knc-content', 'main', '.cms-body', '.article__body']:
            el = soup.select_one(sel)
            if el and len(el.find_all('p')) >= 2:
                block = el
                break
        if not block:
            block = soup.body or soup
        
        # Extract text from paragraphs
        paragraphs = []
        for el in block.find_all(['p', 'h2', 'h3'], recursive=True):
            t = _clean_text(el.get_text(strip=True))
            if t and len(t) > 40:
                paragraphs.append(t)
        
        # Extract images
        images = []
        for el in block.find_all(['figure', 'img'], recursive=True):
            im = el if el.name == 'img' else el.find('img')
            if im:
                src = im.get('data-src') or im.get('src') or im.get('data-original') or ''
                if src and 'base64' not in src:
                    if src.startswith('//'):
                        src = 'https:' + src
                    images.append(src)
        
        # Prefer OG image as main image
        image = og_image or (images[0] if images else '')
        
        return {
            'title': title,
            'summary': paragraphs[0] if paragraphs else '',
            'text': '\n'.join(paragraphs),
            'image': image,
            'og_image': og_image,
            'via': _domain(url),
            'images': images
        }
    except Exception as e:
        return {'title': url, 'summary': '', 'text': '', 'image': '', 'og_image': '', 'via': _domain(url), 'error': str(e)}


def web_context(topic: str, limit: int = 5) -> tuple:
    """Get web context for a topic. Returns (context_text, sources_list)."""
    sources = []
    try:
        # Try Google News RSS
        rss_url = f"https://news.google.com/rss/search?q={quote_plus(topic)}&hl=vi&gl=VN&ceid=VN:vi"
        r = requests.get(rss_url, headers=HEADERS, timeout=15)
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text, 'xml')
        for it in soup.find_all('item')[:limit]:
            title = it.find('title').get_text(' ', strip=True) if it.find('title') else ''
            link = it.find('link').get_text(strip=True) if it.find('link') else ''
            if title and link:
                sources.append({'title': title, 'url': link, 'via': _domain(link)})
    except Exception:
        pass
    
    context = f'Trên mạng có nhiều bài viết về "{topic}". Một số nguồn: ' + ', '.join([s.get('title', '') for s in sources[:3]])
    return context, sources


# ===== SHORT FRAME FUNCTION (required by ai_patch.py) =====
def _make_short_frame(post, img_path, out_path):
    """Create a short video frame from post and image.
    
    Called by ai_patch.py _make_short_frame_full when Image is available.
    """
    if Image is None:
        # Create a minimal frame without PIL - just return success
        # The caller should handle this case
        return False
    
    W, H = 1080, 1920
    bg = Image.new("RGB", (W, H), (14, 14, 14))
    
    try:
        im = Image.open(img_path).convert("RGB")
        target = (1080, 760)
        im_ratio = im.width / max(1, im.height)
        target_ratio = target[0] / target[1]
        
        if im_ratio > target_ratio:
            new_h = target[1]
            new_w = int(new_h * im_ratio)
        else:
            new_w = target[0]
            new_h = int(new_w / im_ratio)
        
        im = im.resize((new_w, new_h))
        left = (new_w - target[0]) // 2
        top = (new_h - target[1]) // 2
        im = im.crop((left, top, left + target[0], top + target[1]))
        bg.paste(im, (0, 0))
    except Exception:
        pass
    
    draw = ImageDraw.Draw(bg)
    
    try:
        font_title = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 54)
        font_body = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 38)
    except Exception:
        font_title = font_body = None
    
    draw.rectangle((0, 720, W, H), fill=(14, 14, 14))
    margin = 48
    maxw = W - margin * 2
    
    y = 830
    for ln in _wrap_text(draw, post.get("title", ""), font_title, maxw, 4):
        draw.text((margin, y), ln, fill=(255, 255, 255), font=font_title)
        y += 66
    
    y += 18
    text = post.get("text", "")
    text = re.sub(r"Nguồn tham khảo:.*", "", text, flags=re.S).strip()
    body_lines = _wrap_text(draw, text, font_body, maxw, 14)
    for ln in body_lines:
        draw.text((margin, y), ln, fill=(220, 220, 220), font=font_body)
        y += 50
        if y > 1640:
            break
    
    bg.save(out_path, quality=92)
    return True


def _wrap_text(draw, text, font, max_width, max_lines):
    """Helper for wrapping text in frames."""
    words = _clean_text(text).split()
    lines, cur = [], ""
    for w in words:
        test = (cur + " " + w).strip()
        try:
            width = draw.textbbox((0, 0), test, font=font)[2]
        except Exception:
            width = len(test) * 20
        if width <= max_width:
            cur = test
        else:
            if cur:
                lines.append(cur)
            cur = w
        if len(lines) >= max_lines:
            break
    if cur and len(lines) < max_lines:
        lines.append(cur)
    return lines