ai_patch.py

import os
import re
import time
import random
import json
import html as html_lib
import subprocess
import requests
import hashlib
from datetime import datetime, timezone
import ai_ext as base
from ai_ext import app
from fastapi import Request
from fastapi.responses import JSONResponse, HTMLResponse, FileResponse
from bs4 import BeautifulSoup
from urllib.parse import quote_plus

try:
    from PIL import Image, ImageDraw, ImageFont
except Exception:
    Image = ImageDraw = ImageFont = None

_STOP = set('và của các những một được trong với cho tại sau trước khi không người vietnam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì này đã để'.split())

def _clean(s):
    """Clean text - replace None with empty string, collapse whitespace."""
    s = s or ''
    s = s.strip()
    s = re.sub(r'\s+', ' ', s)
    return s

def _postprocess_ai_text(text, max_units=20):
    """Clean AI-generated text: remove HTML artifacts, deduplicate, return string with bullet points."""
    text = _clean(text)
    if not text:
        return ""
    # Remove malformed HTML with both href and src attributes (e.g., <a href="..." src="...">)
    text = re.sub(r'<a[^>]*href\s*=\s*"[^"]*"[^>]*\s*src\s*=\s*"[^"]*"[^>]*>', '', text, flags=re.I)
    # Remove any remaining HTML artifacts
    text = re.sub(r'<a\s+[^>]*>', '', text, flags=re.I)
    text = re.sub(r'<img\s+[^>]*>', '', text, flags=re.I)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'</a>', '', text, flags=re.I)
    # Fix truncated HTML tags (missing closing >)
    text = text.replace('<a href=" src="', '').replace("<a href=' src='", '')
    text = text.replace('<a href=" src=', '').replace("<a href=' src=", '')
    # Remove leading bullet/dash
    text = re.sub(r'^\s*[-•]\s*', '', text)
    # Split by double-space (AI often separates items with double space)
    units = text.split('  ')
    out = []
    seen = set()
    for u in units:
        u = _clean(u)
        if not u:
            continue
        # Deduplicate by word signature
        w = tuple(re.findall(r"[\w']+", u.lower()))
        if w in seen:
            continue
        seen.add(w)
        out.append("• " + u)
        if len(out) >= max_units:
            break
    return "\n".join(out) if out else ""

def _scrape_article_images(url):
    """Use the comprehensive scraper from ai_ext for images + paragraphs."""
    sents = []; imgs = []; og = ''
    try:
        if url:
            # Use the comprehensive scraper first
            try:
                data = base.scrape_any_url(url)
                if data and data.get('text'):
                    sents = [p.strip() for p in data['text'].split('\n') if len(p.strip()) > 40]
                    imgs = data.get('images', [])
                    og = data.get('og_image', '')
                    if not imgs and og:
                        imgs = [og]
                    if sents or imgs:
                        return {'paragraphs': sents, 'images': imgs, 'og_img': og}
            except:
                pass
            
            # Fallback: basic scrape
            r = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=8)
            r.encoding = 'utf-8'
            soup = BeautifulSoup(r.text, 'lxml')
            og = soup.find('meta', property='og:image')
            og = og.get('content', '') if og else ''
            if og and og.startswith('//'):
                og = 'https:' + og
            for p in soup.find_all('p'):
                t = _clean(p.get_text())
                if len(t) > 30:
                    sents.append(t)
            seen = set()
            for im in soup.find_all('img'):
                src = im.get('data-src') or im.get('src') or im.get('data-lazy') or ''
                if src and 'base64' not in src and src not in seen:
                    seen.add(src)
                    if src.startswith('//'):
                        src = 'https:' + src
                    imgs.append(src)
            # Filter: prefer large images
            imgs = [im for im in imgs if not any(x in im.lower() for x in ['logo', 'icon', 'avatar', '.svg', 'banner'])]
        return {'paragraphs': sents, 'images': imgs, 'og_img': og}
    except Exception as e:
        return {'paragraphs': [], 'images': [], 'og_img': ''}

def _extract_key_points_for_slides(paragraphs, max_points=8):
    """Extract key points from paragraphs, splitting long ones into sentence-level points."""
    pts = []
    for p in paragraphs:
        if len(p) < 40:
            continue
        s = _clean(p)
        # Split long paragraphs into sentence-level points
        if len(s) > 200:
            sentences = re.split(r'(?<=[.!?])\s+(?=[A-Z0-9À-Ỹ])', s)
            for sent in sentences:
                sent = _clean(sent)
                if len(sent) > 60 and sent not in pts:
                    pts.append(sent)
                    if len(pts) >= max_points:
                        break
        else:
            if s and s not in pts:
                pts.append(s)
        if len(pts) >= max_points:
            break
    return pts

def _make_summary_prompt(title, body, source):
    topic = source or "báo chính"
    return f"""Tóm tắt tin tức: {title}
Nguồn: {topic}
---
{body[:1200] if body else ''}
---
Hãy viết tóm tắt ngắn gọn, súc tích bằng tiếng Việt. Dùng bullet point (•) để liệt kê điểm chính. Không dùng định dạng HTML. Không lặp lại tiêu đề. Không viết "Bản tin VNEWS" hay tiêu đề phụ."""

def _source_line(sources=None):
    if not sources:
        return "Nguồn tham khảo: VNEWS"
    src = sources[0] if sources else {}
    title = src.get('title', '')
    url = src.get('url', '')
    via = src.get('via', '')
    if url and via:
        return f"Nguồn: [{via}]({url})"
    if url:
        return f"Nguồn: {url}"
    if via:
        return f"Nguồn: {via}"
    return "Nguồn tham khảo: VNEWS"

def _summary_segments_from_post(post, max_segments=15):
    text = post.get('text', '')
    segs = []
    if text:
        for line in text.split('\n'):
            line = _clean(line)
            if line:
                segs.append(line)
    return segs[:max_segments]

def _make_scene_frame(post, segment, idx, total, img_path, out_path, emotion='neutral'):
    from PIL import Image, ImageDraw, ImageFont
    sz = (1080, 1920)
    bg = Image.new('RGB', sz, (15, 23, 38))
    d = ImageDraw.Draw(bg)
    try:
        f = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 48)
        fb = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 56)
    except:
        f = ImageFont.load_default()
        fb = f
    # Draw title bar
    d.rectangle([0, 0, sz[0], 120], fill=(25, 118, 210))
    title = post.get('title', '')[:60]
    d.text((sz[0]//2, 60), title, fill='white', font=fb, anchor='mm')
    # Draw segment text
    y = 180
    for line in segment[:120]:
        y += 56
        if y > sz[1] - 100:
            break
        d.text((80, y), line, fill='white', font=f)
    bg.save(out_path, quality=85)

@app.get('/health')
async def health():
    return JSONResponse({'status': 'ok'})