Add auto_scheduler.py - tự động đăng 3 bài rewrite AI + short từ 3 chủ đề HOT lúc 7:00, 13:00, 19:00

auto_scheduler.py

@@ -0,0 +1,361 @@
+"""VNEWS Auto Scheduler - tự động đăng 3 bài rewrite AI + shorts từ 3 chủ đề HOT
+Vào các khung giờ: 7:00, 13:00, 19:00 (giờ Việt Nam)
+Mỗi bài: Rewrite AI từ nguồn báo + short video tự động
+"""
+import os, re, json, time, threading, asyncio, logging, random, hashlib
+from datetime import datetime, timezone, timedelta
+from urllib.parse import quote
+import requests
+from bs4 import BeautifulSoup
+
+VN_TZ = timezone(timedelta(hours=7))
+LOG = logging.getLogger("auto_scheduler")
+LOG.setLevel(logging.INFO)
+if not LOG.handlers:
+    ch = logging.StreamHandler()
+    ch.setFormatter(logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: %(message)s'))
+    LOG.addHandler(ch)
+
+SCHEDULE_TIMES = [(7, 0), (13, 0), (19, 0)]
+
+# ===== Hot topics from VN RSS (same logic as app_v2_entry._get_hot_topics) =====
+_STOP = set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì này đã để'.split())
+
+def _clean(s):
+    return re.sub(r"\s+", " ", str(s or "")).strip()
+
+def _get_hot_topics():
+    freq = {}; display = {}
+    feeds = [
+        'https://vnexpress.net/rss/tin-moi-nhat.rss',
+        'https://dantri.com.vn/rss/home.rss',
+        'https://vietnamnet.vn/rss/tin-moi-nhat.rss',
+        'https://thanhnien.vn/rss/home.rss',
+        'https://tuoitre.vn/rss/tin-moi-nhat.rss',
+        'https://genk.vn/rss',
+        'https://vnexpress.net/rss/the-thao.rss',
+        'https://thethaovanhoa.vn/rss/tin-nong.rss',
+        'https://vnexpress.net/rss/kinh-doanh.rss',
+        'https://dantri.com.vn/rss/the-gioi.rss',
+    ]
+    for feed_url in feeds:
+        try:
+            r = requests.get(feed_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=6)
+            r.encoding = 'utf-8'
+            soup = BeautifulSoup(r.text, 'xml')
+            for item in soup.find_all('item')[:12]:
+                title = _clean(item.find('title').get_text() if item.find('title') else '')
+                if not title: continue
+                title = re.sub(r'\s*[-|].*$', '', title)
+                words = [w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+', title) if len(w) > 2 and w.lower() not in _STOP]
+                if len(words) < 2: continue
+                for n in (3, 4, 2):
+                    for i in range(max(0, len(words) - n + 1)):
+                        phrase = ' '.join(words[i:i + n])
+                        if 8 <= len(phrase) <= 45:
+                            key = phrase.lower()
+                            freq[key] = freq.get(key, 0) + 1
+                            display[key] = phrase
+        except Exception:
+            continue
+    ranked = sorted(freq.items(), key=lambda x: x[1], reverse=True)
+    topics = []; seen = set()
+    for key, count in ranked:
+        kw = display[key]
+        is_dup = any(len(set(e.split()) & set(key.split())) / max(len(set(e.split())), len(set(key.split())), 1) > 0.6 for e in seen)
+        if is_dup: continue
+        seen.add(key)
+        topics.append({'label': '#' + re.sub(r'\s+', '', kw.title()), 'topic': kw, 'count': count})
+        if len(topics) >= 20: break
+    for kw in ['World Cup 2026', 'Kinh tế Việt Nam', 'Bóng đá châu Âu', 'Công nghệ AI', 'Giá vàng', 'Thời tiết']:
+        if len(topics) >= 24: break
+        if not any(kw.lower() in s for s in seen):
+            topics.append({'label': '#' + re.sub(r'\s+', '', kw.title()), 'topic': kw, 'count': 0})
+    return topics[:24]
+
+
+# ===== Import AI functions lazily to avoid circular imports =====
+_ai_ext = None
+_ai_patch = None
+
+def _get_ai_ext():
+    global _ai_ext
+    if _ai_ext is None:
+        import ai_ext as m
+        _ai_ext = m
+    return _ai_ext
+
+def _get_ai_patch():
+    global _ai_patch
+    if _ai_patch is None:
+        import ai_patch as m
+        _ai_patch = m
+    return _ai_patch
+
+
+async def _create_ai_post(topic):
+    """Create one AI rewrite post for a hot topic. Returns list of posts."""
+    ai_ext = _get_ai_ext()
+    ai_patch = _get_ai_patch()
+    
+    articles = ai_patch._topic_source_articles(topic, limit=4)
+    if not articles:
+        LOG.warning(f"No articles found for topic: {topic}")
+        return []
+    
+    posts = []
+    wall = ai_ext._load_ai_wall()
+    if not isinstance(wall, list):
+        wall = []
+    
+    for art in articles:
+        try:
+            prompt = ai_patch._make_summary_prompt(
+                art.get('title', topic),
+                art.get('raw', ''),
+                art.get('via', '')
+            )
+            text = await ai_ext.qwen_generate(prompt, image_url=art.get('image'), max_tokens=1500)
+            text = ai_patch._postprocess_ai_text(text, max_units=20)
+            src = [art.get('source', {'title': art.get('title', topic), 'url': art.get('url', ''), 'via': art.get('via', '')})]
+            if 'Nguồn tham khảo:' not in (text or ''):
+                text = (text or '') + "\n\n" + ai_patch._source_line(src)
+            img = art.get('image') or ai_ext.pollination_image_url(art['title'])
+            post = ai_ext.make_post(art['title'], text, img, art.get('url', ''), 'auto_scheduled', sources=src)
+            
+            # Try to generate slides from article content
+            try:
+                page_data = ai_patch._scrape_article_images(art.get('url', ''))
+                if page_data and page_data.get('paragraphs'):
+                    key_points = ai_patch._extract_key_points_for_slides(page_data['paragraphs'], max_points=8)
+                    if key_points:
+                        relevant_imgs = page_data.get('images', [])
+                        if not relevant_imgs and page_data.get('og_img'):
+                            relevant_imgs = [page_data['og_img']]
+                        slides = []
+                        for i, point in enumerate(key_points):
+                            img_s = relevant_imgs[i] if i < len(relevant_imgs) else (relevant_imgs[-1] if relevant_imgs else '')
+                            slides.append({'text': point, 'image': img_s, 'index': i + 1})
+                        post['slides'] = slides
+            except Exception:
+                pass
+            
+            posts.append(post)
+        except Exception as e:
+            LOG.error(f"Error creating post for article '{art.get('title', '')}': {e}")
+    
+    # Save posts to wall
+    wall = posts + wall
+    ai_ext._save_ai_wall(wall)
+    
+    # Try to generate short video for each post
+    for post in posts:
+        try:
+            _try_generate_short(post)
+        except Exception as e:
+            LOG.warning(f"Short generation skipped for '{post.get('title', '')[:40]}': {e}")
+    
+    return posts
+
+
+def _try_generate_short(post):
+    """Try to generate a short video for a post. Uses gTTS + ffmpeg."""
+    post_id = post.get('id', '')
+    if not post_id:
+        return
+    
+    try:
+        ai_ext = _get_ai_ext()
+        ai_patch = _get_ai_patch()
+        if ai_ext.gTTS is None:
+            LOG.warning("gTTS not available, skipping short generation")
+            return
+        
+        segments = ai_patch._summary_segments_from_post(post, max_segments=15)
+        if not segments:
+            return
+        
+        seg_hash = hashlib.md5(('|'.join(segments) + 'nu' + 'neutral' + '1.0').encode('utf-8')).hexdigest()[:8]
+        suffix = f"_nu_neutral_1p0_{seg_hash}_scenes_nosub"
+        out_mp4 = os.path.join(ai_ext.SHORTS_DIR, ai_ext._safe_name(post_id + suffix) + '.mp4')
+        
+        if os.path.exists(out_mp4):
+            post['video'] = '/api/ai/short-file/' + post_id + suffix
+            wall = ai_ext._load_ai_wall()
+            for i, p in enumerate(wall):
+                if p.get('id') == post_id:
+                    wall[i] = post
+                    break
+            ai_ext._save_ai_wall(wall)
+            return
+        
+        # Generate short in background using threading
+        threading.Thread(
+            target=_generate_short_worker,
+            args=(post, segments, post_id, suffix, out_mp4),
+            daemon=True
+        ).start()
+        
+    except Exception as e:
+        LOG.warning(f"Short generation init failed: {e}")
+
+
+def _generate_short_worker(post, segments, post_id, suffix, out_mp4):
+    """Worker thread to generate short video."""
+    import subprocess
+    try:
+        ai_ext = _get_ai_ext()
+        ai_patch = _get_ai_patch()
+        work = os.path.join(ai_ext.SHORTS_DIR, ai_ext._safe_name(post_id + suffix))
+        os.makedirs(work, exist_ok=True)
+        
+        img = os.path.join(work, 'image.jpg')
+        ai_ext._download_image(post.get('img'), post.get('title', 'AI news'), img)
+        
+        part_files = []
+        for idx, seg in enumerate(segments[:10]):
+            frame = os.path.join(work, f'frame_{idx:02d}.jpg')
+            aud = os.path.join(work, f'voice_{idx:02d}.mp3')
+            aud_fast = os.path.join(work, f'voice_{idx:02d}_fast.mp3')
+            part = os.path.join(work, f'part_{idx:02d}.mp4')
+            try:
+                ai_patch._make_scene_frame(post, seg, idx, min(len(segments), 10), img, frame, emotion='neutral')
+            except Exception:
+                if not os.path.exists(img):
+                    continue
+                from PIL import Image
+                bg = Image.new('RGB', (1080, 1920), (14, 14, 14))
+                bg.save(frame, quality=85)
+            tts_text = re.sub(r'^[•\-\*\d\.\)\s]+', '', seg).strip()
+            try:
+                ai_ext.gTTS(tts_text, lang='vi', slow=False).save(aud)
+            except Exception:
+                try:
+                    ai_ext.gTTS(tts_text, lang='vi', tld='com.vn', slow=False).save(aud)
+                except Exception:
+                    continue
+            subprocess.run(['ffmpeg', '-y', '-i', aud, '-filter:a', 'atempo=1.0', '-vn', aud_fast],
+                           check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=90)
+            dur = 12.0
+            try:
+                pr = subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration',
+                                     '-of', 'default=noprint_wrappers=1:no_key=1', aud_fast],
+                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20)
+                dur = max(8.0, float((pr.stdout or b'').decode().strip() or 12.0)) + 0.5
+            except Exception:
+                pass
+            subprocess.run(['ffmpeg', '-y', '-loop', '1', '-t', str(dur), '-i', frame,
+                           '-i', aud_fast, '-shortest', '-c:v', 'libx264', '-tune', 'stillimage',
+                           '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '128k', part],
+                          check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=150)
+            part_files.append(part)
+        
+        if part_files:
+            concat = os.path.join(work, 'concat.txt')
+            with open(concat, 'w', encoding='utf-8') as f:
+                for p in part_files:
+                    f.write("file '" + p.replace("'", "'\\''") + "'\n")
+            subprocess.run(['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', concat, '-c', 'copy', out_mp4],
+                           check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180)
+            post['video'] = '/api/ai/short-file/' + post_id + suffix
+            post['short_voice'] = 'nu'
+            post['short_emotion'] = 'neutral'
+            post['short_speed'] = 1.0
+            post['short_segments'] = segments
+            post['short_subtitles'] = False
+            wall = ai_ext._load_ai_wall()
+            for i, p in enumerate(wall):
+                if p.get('id') == post_id:
+                    wall[i] = post
+                    break
+            ai_ext._save_ai_wall(wall)
+            LOG.info(f"Short video generated for post: {post_id}")
+    except Exception as e:
+        LOG.warning(f"Short video generation failed for {post_id}: {e}")
+
+
+def _run_scheduled_posting():
+    """Run the scheduled posting task - creates 3 AI rewrite posts from 3 hot topics."""
+    LOG.info("=" * 50)
+    LOG.info("Auto scheduler triggered at %s", datetime.now(VN_TZ).strftime('%H:%M %d/%m/%Y'))
+    LOG.info("=" * 50)
+    
+    try:
+        hot_topics = _get_hot_topics()
+        if not hot_topics:
+            LOG.warning("No hot topics found")
+            return
+        
+        selected = []
+        seen_labels = set()
+        for t in hot_topics:
+            label = t.get('label', '')
+            if label and label not in seen_labels:
+                seen_labels.add(label)
+                selected.append(t['topic'])
+            if len(selected) >= 3:
+                break
+        
+        if len(selected) < 3:
+            selected = ['Thời sự Việt Nam', 'Kinh tế Việt Nam', 'Thể thao']
+        
+        LOG.info(f"Selected {len(selected)} hot topics: {selected}")
+        
+        async def _do_all():
+            all_results = []
+            for topic in selected:
+                try:
+                    posts = await _create_ai_post(topic)
+                    if posts:
+                        all_results.append({'topic': topic, 'posts': len(posts)})
+                        LOG.info(f"✓ Created {len(posts)} post(s) for topic: '{topic}'")
+                    else:
+                        LOG.warning(f"✗ No posts created for topic: '{topic}'")
+                except Exception as e:
+                    LOG.error(f"✗ Error creating post for topic '{topic}': {e}")
+            return all_results
+        
+        results = asyncio.run(_do_all())
+        
+        LOG.info("-" * 40)
+        LOG.info(f"Auto scheduler completed: {len(results)} topic(s) processed")
+        for r in results:
+            LOG.info(f"  • {r['topic']}: {r['posts']} bài đã đăng")
+        LOG.info("=" * 50)
+        
+    except Exception as e:
+        LOG.error(f"Auto scheduler error: {e}")
+
+
+def _scheduler_loop():
+    """Background thread that checks every minute and runs at scheduled times."""
+    LOG.info("🕐 Auto scheduler background thread started")
+    LOG.info(f"⏰ Schedule: {', '.join(f'{h:02d}:{m:02d}' for h, m in SCHEDULE_TIMES)} (VN time)")
+    last_run_dates = {t: None for t in SCHEDULE_TIMES}
+    
+    while True:
+        try:
+            now = datetime.now(VN_TZ)
+            current_key = (now.hour, now.minute)
+            
+            for sched_time in SCHEDULE_TIMES:
+                if current_key == sched_time:
+                    last_date = last_run_dates.get(sched_time)
+                    if last_date != now.date():
+                        last_run_dates[sched_time] = now.date()
+                        LOG.info(f"⏰ Scheduled run at {now.hour:02d}:{now.minute:02d}")
+                        _run_scheduled_posting()
+                    break
+            
+            time.sleep(60)
+        except Exception as e:
+            LOG.error(f"Scheduler loop error: {e}")
+            time.sleep(60)
+
+
+def start_auto_scheduler():
+    """Start the auto scheduler in a background daemon thread."""
+    thread = threading.Thread(target=_scheduler_loop, daemon=True, name="auto-scheduler")
+    thread.start()
+    LOG.info("🚀 Auto scheduler started successfully")
+    return thread