Restore ai_runtime_fix.py from target commit 0280ca57

ai_runtime_fix.py

@@ -0,0 +1,378 @@
+"""VNEWS Short Video Fix - standalone module with clean registration.
+This module MUST be imported LAST to register /api/ai/short endpoints.
+FIX v1: No route filtering issues - registers endpoints unconditionally.
+"""
+import os
+import re
+import time
+import json
+import sys
+import logging
+import asyncio
+import hashlib
+import subprocess
+import requests
+from datetime import datetime, timezone, timedelta
+from urllib.parse import urlparse
+from fastapi import Request, Query
+from fastapi.responses import JSONResponse, FileResponse
+
+# Import dependencies
+try:
+    import ai_ext as base
+except ImportError:
+    import ai_runtime_final6 as base
+
+# Try to import app from various sources
+try:
+    from app_v2_entry import app
+except ImportError:
+    try:
+        from main import app
+    except ImportError:
+        from ai_runtime_final6 import app
+
+_log = logging.getLogger("short_fix")
+_log.setLevel(logging.INFO)
+if not _log.handlers:
+    _log.addHandler(logging.StreamHandler(sys.stderr))
+
+DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
+os.makedirs(DATA_DIR, exist_ok=True)
+SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts")
+os.makedirs(SHORTS_DIR, exist_ok=True)
+
+# ===== VIETNAMESE FONT DETECTION =====
+_VN_FONT_REG = None
+_VN_FONT_BOLD = None
+
+def _get_vn_fonts():
+    """Find Vietnamese-supporting fonts."""
+    global _VN_FONT_REG, _VN_FONT_BOLD
+    if _VN_FONT_REG is not None:
+        return _VN_FONT_REG, _VN_FONT_BOLD
+    
+    try:
+        from PIL import ImageFont
+    except Exception:
+        _log.error("PIL not available!")
+        return None, None
+    
+    # Priority: Noto > DejaVu > Liberation
+    reg_paths = [
+        "/usr/share/fonts/truetype/noto/NotoSans-Regular.ttf",
+        "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+        "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
+        "/usr/share/fonts/truetype/freefont/FreeSans.ttf",
+    ]
+    bold_paths = [
+        "/usr/share/fonts/truetype/noto/NotoSans-Bold.ttf",
+        "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
+        "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
+        "/usr/share/fonts/truetype/freefont/FreeSans.ttf",
+    ]
+    
+    for path in reg_paths:
+        if os.path.exists(path):
+            try:
+                _VN_FONT_REG = ImageFont.truetype(path, 40)
+                _log.info(f"Found regular font: {path}")
+                break
+            except:
+                continue
+    
+    for path in bold_paths:
+        if os.path.exists(path):
+            try:
+                _VN_FONT_BOLD = ImageFont.truetype(path, 52)
+                _log.info(f"Found bold font: {path}")
+                break
+            except:
+                continue
+    
+    if _VN_FONT_REG is None:
+        _VN_FONT_REG = ImageFont.load_default()
+    if _VN_FONT_BOLD is None:
+        _VN_FONT_BOLD = _VN_FONT_REG
+    
+    return _VN_FONT_REG, _VN_FONT_BOLD
+
+
+def _clean(s):
+    import html as html_lib
+    return re.sub(r"\s+", " ", html_lib.unescape(str(s or ""))).strip()
+
+
+# ===== ROBUST TEXT SEGMENTATION =====
+def _split_into_segments(text, max_segments=10, min_len=30):
+    """Split text into segments - multi strategy."""
+    text = _clean(text)
+    if not text:
+        return []
+    
+    # Strategy 1: bullet points
+    lines = text.split('\n')
+    segmented = []
+    for line in lines:
+        line = _clean(line)
+        line_bare = re.sub(r'^[•\-\*\d\.\)\s]+', '', line).strip()
+        if len(line_bare) > min_len:
+            segmented.append(line_bare)
+        elif len(line) > min_len:
+            segmented.append(line)
+    
+    # Strategy 2: sentences (Vietnamese)
+    if len(segmented) < 2:
+        sents = re.split(r'(?<=[.!?])\s+(?=[A-Z0-9À-ỸĐ])', text)
+        segmented = [s for s in sents if len(_clean(s)) > min_len]
+    
+    # Strategy 3: character chunks
+    if not segmented:
+        words = text.split()
+        for i in range(0, min(len(words), max_segments * 20), 20):
+            chunk = ' '.join(words[i:i+20])
+            if len(chunk) > min_len:
+                segmented.append(chunk)
+    
+    # Strategy 4: fallback
+    if not segmented:
+        segmented = [text[:300]]
+    
+    return segmented[:max_segments]
+
+
+# ===== SHORT VIDEO GENERATOR =====
+def _gen_short_core(post, work_dir):
+    """Core short generation - returns video path or None."""
+    post_id = post.get('id', '')
+    text = post.get('text', '') or post.get('title', '')
+    
+    if not post_id or len(text) < 100:
+        _log.error(f"Invalid post: id={post_id}, text_len={len(text)}")
+        return None
+    
+    segments = _split_into_segments(text, max_segments=10, min_len=30)
+    if not segments:
+        _log.error("No segments generated")
+        return None
+    
+    _log.info(f"Generating short: {len(segments)} segments")
+    
+    seg_hash = hashlib.md5(('|'.join(segments) + 'nu').encode()).hexdigest()[:8]
+    suffix = f"_nu_{seg_hash}"
+    out_mp4 = os.path.join(work_dir, f"{post_id}{suffix}.mp4")
+    
+    if os.path.exists(out_mp4):
+        _log.info(f"Already exists: {out_mp4}")
+        return out_mp4
+    
+    # Check dependencies
+    try:
+        subprocess.run(['ffmpeg', '-version'], capture_output=True, timeout=5)
+    except Exception as e:
+        _log.error(f"ffmpeg missing: {e}")
+        return None
+    
+    # Download image
+    img_path = os.path.join(work_dir, 'bg.jpg')
+    downloaded = False
+    try:
+        img_url = post.get('img', '')
+        if img_url and img_url.startswith('http'):
+            r = requests.get(img_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=12)
+            if r.status_code == 200:
+                with open(img_path, 'wb') as f:
+                    f.write(r.content)
+                downloaded = True
+    except Exception as e:
+        _log.warning(f"Image download: {e}")
+    
+    try:
+        from PIL import Image, ImageDraw
+        has_pil = True
+    except:
+        has_pil = False
+        _log.warning("PIL not available")
+    
+    try:
+        from gtts import gTTS
+        has_tts = True
+    except:
+        has_tts = False
+        _log.warning("gTTS not available")
+    
+    parts = []
+    
+    for i, seg in enumerate(segments[:10]):
+        frame = os.path.join(work_dir, f'frame_{i}.jpg')
+        audio = os.path.join(work_dir, f'audio_{i}.mp3')
+        part = os.path.join(work_dir, f'part_{i}.mp4')
+        
+        # Create frame
+        try:
+            if has_pil:
+                _make_frame(post, seg, img_path, downloaded, frame)
+            else:
+                subprocess.run(['ffmpeg', '-y', '-f', 'lavfi', '-i',
+                    'color=c=black:s=1080x1920:d=1', '-frames:v', '1', frame],
+                    capture_output=True, timeout=20)
+        except Exception as e:
+            _log.error(f"Frame error: {e}")
+            continue
+        
+        # Create audio
+        if has_tts:
+            try:
+                tts = _clean(seg)[:300]
+                gTTS(tts, lang='vi', slow=False).save(audio)
+            except Exception as e:
+                _log.warning(f"TTS error: {e}")
+                audio = None
+        
+        # Combine
+        dur = 10
+        try:
+            cmd = ['ffmpeg', '-y', '-loop', '1', '-t', str(dur), '-i', frame]
+            if has_tts and os.path.exists(audio):
+                cmd += ['-i', audio, '-shortest']
+            else:
+                cmd += ['-f', 'lavfi', '-i', 'anullsrc', '-shortest']
+            cmd += ['-c:v', 'libx264', '-tune', 'stillimage', '-pix_fmt', 'yuv420p',
+                    '-c:a', 'aac', '-b:a', '128k', part]
+            subprocess.run(cmd, capture_output=True, timeout=120)
+            if os.path.exists(part) and os.path.getsize(part) > 5000:
+                parts.append(part)
+        except Exception as e:
+            _log.error(f"Part combine error: {e}")
+    
+    if not parts:
+        _log.error("No video parts created!")
+        return None
+    
+    # Concatenate
+    try:
+        concat = os.path.join(work_dir, 'list.txt')
+        with open(concat, 'w') as f:
+            for p in parts:
+                f.write(f"file '{p}'\n")
+        subprocess.run(['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', concat, '-c', 'copy', out_mp4],
+            capture_output=True, timeout=180)
+        _log.info(f"Short created: {out_mp4}")
+        return out_mp4
+    except Exception as e:
+        _log.error(f"Concat error: {e}")
+        return None
+
+
+def _make_frame(post, text, img_path, downloaded, out_path):
+    """Create video frame with Vietnamese font."""
+    from PIL import Image, ImageDraw
+    _get_vn_fonts()
+    
+    W, H = 1080, 1920
+    bg = Image.new('RGB', (W, H), (15, 23, 38))
+    d = ImageDraw.Draw(bg)
+    
+    # Background image
+    if downloaded and os.path.exists(img_path):
+        try:
+            im = Image.open(img_path).convert('RGB')
+            im = im.resize((W, 760))
+            bg.paste(im, (0, 0))
+        except:
+            pass
+    
+    # Title
+    d.rectangle([0, 0, W, 100], fill=(25, 118, 210))
+    ttl = post.get('title', '')[:50]
+    if _VN_FONT_BOLD:
+        d.text((W//2, 50), ttl, fill='white', font=_VN_FONT_BOLD, anchor='mm')
+    
+    # Content
+    y = 150
+    for ln in _wrap_text(d, text[:200], _VN_FONT_REG, 80, 920, 10):
+        d.text((80, y), ln, fill='white', font=_VN_FONT_REG)
+        y += 55
+    
+    bg.save(out_path, quality=85)
+
+
+def _wrap_text(draw, text, font, x, max_w, max_lines):
+    """Word wrap text."""
+    words = text.split()
+    lines = []
+    cur = []
+    for w in words:
+        test = ' '.join(cur + [w])
+        try:
+            w_px = draw.textbbox((0, 0), test, font=font)[2]
+        except:
+            w_px = len(test) * 22
+        if w_px <= max_w:
+            cur.append(w)
+        else:
+            if cur:
+                lines.append(' '.join(cur))
+            cur = [w]
+        if len(lines) >= max_lines:
+            break
+    if cur and len(lines) < max_lines:
+        lines.append(' '.join(cur))
+    return lines
+
+
+def _gen_short_sync(post) -> str:
+    """Sync wrapper - returns video URL."""
+    work = os.path.join(SHORTS_DIR, f"work_{post.get('id', int(time.time()))}")
+    os.makedirs(work, exist_ok=True)
+    result = _gen_short_core(post, work)
+    if result:
+        # Update wall
+        try:
+            wall = base._load_ai_wall()
+            for i, p in enumerate(wall):
+                if str(p.get('id')) == str(post.get('id')):
+                    p['video'] = f'/api/ai/short-file/{post.get("id")}_nu_{hashlib.md5(str(post).encode()).hexdigest()[:8]}'
+                    wall[i] = p
+                    break
+            base._save_ai_wall(wall)
+        except Exception as e:
+            _log.warning(f"Wall update: {e}")
+        return result
+    return ''
+
+
+# ===== REGISTER ENDPOINTS - MUST BE AT MODULE LEVEL =====
+@app.post('/api/ai/short/{post_id}')
+async def api_short_generate(post_id: str, request: Request):
+    _log.info(f"POST /api/ai/short/{post_id}")
+    wall = base._load_ai_wall()
+    post = next((p for p in wall if str(p.get('id')) == str(post_id)), None)
+    if not post:
+        return JSONResponse({'error': 'Post not found in wall'}, status_code=404)
+    
+    if post.get('video'):
+        return JSONResponse({'post': post, 'video': post['video'], 'status': 'done'})
+    
+    loop = asyncio.get_event_loop()
+    result = await loop.run_in_executor(None, _gen_short_sync, post)
+    
+    if result:
+        # Get the video URL from wall (updated in _gen_short_sync)
+        wall = base._load_ai_wall()
+        post = next((p for p in wall if str(p.get('id')) == str(post_id)), post)
+        return JSONResponse({'post': post, 'video': post.get('video'), 'status': 'done'})
+    return JSONResponse({'error': 'Video generation failed'}, status_code=500)
+
+
+@app.get('/api/ai/short-file/{file_id:path}')
+async def api_short_file(file_id: str):
+    safe = re.sub(r'[^\w\-.]', '_', file_id)[:100]
+    for fname in os.listdir(SHORTS_DIR) if os.path.isdir(SHORTS_DIR) else []:
+        if fname.endswith('.mp4') and safe in fname:
+            return FileResponse(os.path.join(SHORTS_DIR, fname), media_type='video/mp4')
+    return JSONResponse({'error': 'Not found'}, status_code=404)
+
+
+# Log startup
+_log.info("Short video endpoints registered")