"""VNEWS Short Video Fix - standalone module with clean registration. This module MUST be imported LAST to register /api/ai/short endpoints. FIX v1: No route filtering issues - registers endpoints unconditionally. FIX v2: SSE inline endpoint for auto homepage updates """ import os import re import time import json import sys import logging import asyncio import hashlib import subprocess import requests from datetime import datetime, timezone, timedelta from urllib.parse import urlparse from fastapi import Request, Query from fastapi.responses import JSONResponse, FileResponse # Import dependencies try: import ai_ext as base except ImportError: import ai_runtime_final6 as base # Try to import app from various sources try: from app_v2_entry import app except ImportError: try: from main import app except ImportError: from ai_runtime_final6 import app _log = logging.getLogger("short_fix") _log.setLevel(logging.INFO) if not _log.handlers: _log.addHandler(logging.StreamHandler(sys.stderr)) DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data" os.makedirs(DATA_DIR, exist_ok=True) SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts") os.makedirs(SHORTS_DIR, exist_ok=True) # ===== VIETNAMESE FONT DETECTION ===== _VN_FONT_REG = None _VN_FONT_BOLD = None def _get_vn_fonts(): """Find Vietnamese-supporting fonts.""" global _VN_FONT_REG, _VN_FONT_BOLD if _VN_FONT_REG is not None: return _VN_FONT_REG, _VN_FONT_BOLD try: from PIL import ImageFont except Exception: _log.error("PIL not available!") return None, None # Priority: Noto > DejaVu > Liberation reg_paths = [ "/usr/share/fonts/truetype/noto/NotoSans-Regular.ttf", "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf", "/usr/share/fonts/truetype/freefont/FreeSans.ttf", ] bold_paths = [ "/usr/share/fonts/truetype/noto/NotoSans-Bold.ttf", "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf", "/usr/share/fonts/truetype/freefont/FreeSans.ttf", ] for path in reg_paths: if os.path.exists(path): try: _VN_FONT_REG = ImageFont.truetype(path, 40) _log.info(f"Found regular font: {path}") break except: continue for path in bold_paths: if os.path.exists(path): try: _VN_FONT_BOLD = ImageFont.truetype(path, 52) _log.info(f"Found bold font: {path}") break except: continue if _VN_FONT_REG is None: _VN_FONT_REG = ImageFont.load_default() if _VN_FONT_BOLD is None: _VN_FONT_BOLD = _VN_FONT_REG return _VN_FONT_REG, _VN_FONT_BOLD def _clean(s): import html as html_lib return re.sub(r"\s+", " ", html_lib.unescape(str(s or ""))).strip() # ===== ROBUST TEXT SEGMENTATION ===== def _split_into_segments(text, max_segments=10, min_len=30): """Split text into segments - multi strategy.""" text = _clean(text) if not text: return [] # Strategy 1: bullet points lines = text.split('\n') segmented = [] for line in lines: line = _clean(line) line_bare = re.sub(r'^[•\-\*\d\.\)\s]+', '', line).strip() if len(line_bare) > min_len: segmented.append(line_bare) elif len(line) > min_len: segmented.append(line) # Strategy 2: sentences (Vietnamese) if len(segmented) < 2: sents = re.split(r'(?<=[.!?])\s+(?=[A-Z0-9À-ỸĐ])', text) segmented = [s for s in sents if len(_clean(s)) > min_len] # Strategy 3: character chunks if not segmented: words = text.split() for i in range(0, min(len(words), max_segments * 20), 20): chunk = ' '.join(words[i:i+20]) if len(chunk) > min_len: segmented.append(chunk) # Strategy 4: fallback if not segmented: segmented = [text[:300]] return segmented[:max_segments] # ===== SHORT VIDEO GENERATOR ===== def _gen_short_core(post, work_dir): """Core short generation - returns video path or None.""" post_id = post.get('id', '') text = post.get('text', '') or post.get('title', '') if not post_id or len(text) < 100: _log.error(f"Invalid post: id={post_id}, text_len={len(text)}") return None segments = _split_into_segments(text, max_segments=10, min_len=30) if not segments: _log.error("No segments generated") return None _log.info(f"Generating short: {len(segments)} segments") seg_hash = hashlib.md5(('|'.join(segments) + 'nu').encode()).hexdigest()[:8] suffix = f"_nu_{seg_hash}" out_mp4 = os.path.join(work_dir, f"{post_id}{suffix}.mp4") if os.path.exists(out_mp4): _log.info(f"Already exists: {out_mp4}") return out_mp4 # Check dependencies try: subprocess.run(['ffmpeg', '-version'], capture_output=True, timeout=5) except Exception as e: _log.error(f"ffmpeg missing: {e}") return None # Download image img_path = os.path.join(work_dir, 'bg.jpg') downloaded = False try: img_url = post.get('img', '') if img_url and img_url.startswith('http'): r = requests.get(img_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=12) if r.status_code == 200: with open(img_path, 'wb') as f: f.write(r.content) downloaded = True except Exception as e: _log.warning(f"Image download: {e}") try: from PIL import Image, ImageDraw has_pil = True except: has_pil = False _log.warning("PIL not available") try: from gtts import gTTS has_tts = True except: has_tts = False _log.warning("gTTS not available") parts = [] for i, seg in enumerate(segments[:10]): frame = os.path.join(work_dir, f'frame_{i}.jpg') audio = os.path.join(work_dir, f'audio_{i}.mp3') part = os.path.join(work_dir, f'part_{i}.mp4') # Create frame try: if has_pil: _make_frame(post, seg, img_path, downloaded, frame) else: subprocess.run(['ffmpeg', '-y', '-f', 'lavfi', '-i', 'color=c=black:s=1080x1920:d=1', '-frames:v', '1', frame], capture_output=True, timeout=20) except Exception as e: _log.error(f"Frame error: {e}") continue # Create audio if has_tts: try: tts = _clean(seg)[:300] gTTS(tts, lang='vi', slow=False).save(audio) except Exception as e: _log.warning(f"TTS error: {e}") audio = None # Combine dur = 10 try: cmd = ['ffmpeg', '-y', '-loop', '1', '-t', str(dur), '-i', frame] if has_tts and os.path.exists(audio): cmd += ['-i', audio, '-shortest'] else: cmd += ['-f', 'lavfi', '-i', 'anullsrc', '-shortest'] cmd += ['-c:v', 'libx264', '-tune', 'stillimage', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '128k', part] subprocess.run(cmd, capture_output=True, timeout=120) if os.path.exists(part) and os.path.getsize(part) > 5000: parts.append(part) except Exception as e: _log.error(f"Part combine error: {e}") if not parts: _log.error("No video parts created!") return None # Concatenate try: concat = os.path.join(work_dir, 'list.txt') with open(concat, 'w') as f: for p in parts: f.write(f"file '{p}'\n") subprocess.run(['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', concat, '-c', 'copy', out_mp4], capture_output=True, timeout=180) _log.info(f"Short created: {out_mp4}") return out_mp4 except Exception as e: _log.error(f"Concat error: {e}") return None def _make_frame(post, text, img_path, downloaded, out_path): """Create video frame with Vietnamese font.""" from PIL import Image, ImageDraw _get_vn_fonts() W, H = 1080, 1920 bg = Image.new('RGB', (W, H), (15, 23, 38)) d = ImageDraw.Draw(bg) # Background image if downloaded and os.path.exists(img_path): try: im = Image.open(img_path).convert('RGB') im = im.resize((W, 760)) bg.paste(im, (0, 0)) except: pass # Title d.rectangle([0, 0, W, 100], fill=(25, 118, 210)) ttl = post.get('title', '')[:50] if _VN_FONT_BOLD: d.text((W//2, 50), ttl, fill='white', font=_VN_FONT_BOLD, anchor='mm') # Content y = 150 for ln in _wrap_text(d, text[:200], _VN_FONT_REG, 80, 920, 10): d.text((80, y), ln, fill='white', font=_VN_FONT_REG) y += 55 bg.save(out_path, quality=85) def _wrap_text(draw, text, font, x, max_w, max_lines): """Word wrap text.""" words = text.split() lines = [] cur = [] for w in words: test = ' '.join(cur + [w]) try: w_px = draw.textbbox((0, 0), test, font=font)[2] except: w_px = len(test) * 22 if w_px <= max_w: cur.append(w) else: if cur: lines.append(' '.join(cur)) cur = [w] if len(lines) >= max_lines: break if cur and len(lines) < max_lines: lines.append(' '.join(cur)) return lines def _gen_short_sync(post) -> str: """Sync wrapper - returns video URL.""" work = os.path.join(SHORTS_DIR, f"work_{post.get('id', int(time.time()))}") os.makedirs(work, exist_ok=True) result = _gen_short_core(post, work) if result: # Update wall try: wall = base._load_ai_wall() for i, p in enumerate(wall): if str(p.get('id')) == str(post.get('id')): p['video'] = f'/api/ai/short-file/{post.get("id")}_nu_{hashlib.md5(str(post).encode()).hexdigest()[:8]}' wall[i] = p break base._save_ai_wall(wall) # Notify SSE for auto-update try: from auto_update_sse import notify_new_short notify_new_short(post) except: pass except Exception as e: _log.warning(f"Wall update: {e}") return result return '' # ===== REGISTER ENDPOINTS - MUST BE AT MODULE LEVEL ===== @app.post('/api/ai/short/{post_id}') async def api_short_generate(post_id: str, request: Request): _log.info(f"POST /api/ai/short/{post_id}") wall = base._load_ai_wall() post = next((p for p in wall if str(p.get('id')) == str(post_id)), None) if not post: return JSONResponse({'error': 'Post not found in wall'}, status_code=404) if post.get('video'): return JSONResponse({'post': post, 'video': post['video'], 'status': 'done'}) loop = asyncio.get_event_loop() result = await loop.run_in_executor(None, _gen_short_sync, post) if result: # Get the video URL from wall (updated in _gen_short_sync) wall = base._load_ai_wall() post = next((p for p in wall if str(p.get('id')) == str(post_id)), post) return JSONResponse({'post': post, 'video': post.get('video'), 'status': 'done'}) return JSONResponse({'error': 'Video generation failed'}, status_code=500) @app.get('/api/ai/short-file/{file_id:path}') async def api_short_file(file_id: str): safe = re.sub(r'[^\w\-.]', '_', file_id)[:100] for fname in os.listdir(SHORTS_DIR) if os.path.isdir(SHORTS_DIR) else []: if fname.endswith('.mp4') and safe in fname: return FileResponse(os.path.join(SHORTS_DIR, fname), media_type='video/mp4') return JSONResponse({'error': 'Not found'}, status_code=404) # ===== SSE ENDPOINT FOR AUTO-UPDATE ===== try: from auto_update_sse import sse_events as _sse_handler app.add_api_route('/api/events', _sse_handler, methods=['GET']) _log.info("SSE endpoint registered at /api/events") except Exception as e: _log.warning(f"SSE route not loaded: {e}") # Log startup _log.info("Short video endpoints registered")