Upload ai_fix2.py

ai_fix2.py

@@ -0,0 +1,356 @@
+import os, re, subprocess, html as html_lib, json
+from urllib.parse import quote_plus, urlparse, parse_qs, unquote
+import requests
+import ai_patch as prev
+from ai_patch import app
+from fastapi import Request
+from fastapi.responses import JSONResponse, HTMLResponse, FileResponse
+
+base = prev.base
+
+
+def clean(s):
+    return re.sub(r"\s+", " ", html_lib.unescape(s or "")).strip()
+
+
+def _is_real_article_text(raw):
+    raw = clean(raw)
+    if len(raw) < 500:
+        return False
+    sentences = re.split(r"(?<=[\.\!\?])\s+", raw)
+    long_sentences = [s for s in sentences if len(s) > 45]
+    return len(long_sentences) >= 5
+
+
+def _extract_ddg_url(href):
+    if not href:
+        return ""
+    if href.startswith("//"):
+        href = "https:" + href
+    if "duckduckgo.com/l/" in href:
+        try:
+            qs = parse_qs(urlparse(href).query)
+            if qs.get("uddg"):
+                return unquote(qs["uddg"][0])
+        except Exception:
+            pass
+    return href
+
+
+def _ddg_article_urls(topic, limit=12):
+    urls = []
+    try:
+        q = quote_plus(topic + " tin tức bài viết phân tích")
+        r = requests.get("https://html.duckduckgo.com/html/?q=" + q, headers=base.HEADERS, timeout=18)
+        r.encoding = "utf-8"
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(r.text, "lxml")
+        for a in soup.select("a.result__a"):
+            u = _extract_ddg_url(a.get("href", ""))
+            if not u.startswith("http"):
+                continue
+            if any(bad in u for bad in ["google.com", "youtube.com", "facebook.com", "x.com", "twitter.com"]):
+                continue
+            if u not in urls:
+                urls.append(u)
+            if len(urls) >= limit:
+                break
+    except Exception:
+        pass
+    return urls
+
+
+def _rss_article_urls(topic, limit=10):
+    out = []
+    try:
+        url = "https://news.google.com/rss/search?q=" + quote_plus(topic) + "&hl=vi&gl=VN&ceid=VN:vi"
+        r = requests.get(url, headers=base.HEADERS, timeout=15)
+        r.encoding = "utf-8"
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(r.text, "xml")
+        for it in soup.find_all("item")[:limit]:
+            title = it.find("title").get_text(" ", strip=True) if it.find("title") else ""
+            link = it.find("link").get_text(strip=True) if it.find("link") else ""
+            src = it.find("source").get_text(" ", strip=True) if it.find("source") else base._domain(link)
+            if title and link:
+                out.append({"title": title, "url": link, "via": src, "excerpt": title})
+    except Exception:
+        pass
+    return out
+
+
+def _topic_source_articles(topic, limit=5):
+    candidates = []
+    seen = set()
+    for u in _ddg_article_urls(topic, limit=14):
+        if u not in seen:
+            seen.add(u)
+            candidates.append({"url": u, "title": "", "via": base._domain(u)})
+    try:
+        _ctx, srcs = base.web_context(topic, limit=8)
+        for s in srcs or []:
+            u = s.get("url") or ""
+            if u.startswith("http") and u not in seen:
+                seen.add(u)
+                candidates.append(s)
+    except Exception:
+        pass
+    for s in _rss_article_urls(topic, limit=10):
+        u = s.get("url") or ""
+        if u.startswith("http") and u not in seen:
+            seen.add(u)
+            candidates.append(s)
+
+    out = []
+    for s in candidates[:24]:
+        url = s.get("url") or ""
+        try:
+            page = base.scrape_any_url(url)
+            raw = (page.get("summary", "") + "\n" + page.get("text", "")).strip()
+            if not _is_real_article_text(raw):
+                continue
+            title = page.get("title") or s.get("title") or url
+            via = page.get("via") or s.get("via") or base._domain(url)
+            out.append({
+                "title": title,
+                "url": url,
+                "raw": raw,
+                "image": page.get("image") or "",
+                "via": via,
+                "source": {"title": title, "url": url, "excerpt": raw[:700], "via": via}
+            })
+            if len(out) >= limit:
+                break
+        except Exception:
+            continue
+    return out[:limit]
+
+
+def sentence_split(text):
+    text = re.sub(r"^[•\-\*]\s*", "", text or "", flags=re.M)
+    text = re.sub(r"\n+", ". ", text)
+    parts = []
+    for s in re.split(r"(?<=[\.\!\?])\s+", text):
+        s = clean(s)
+        if len(s) >= 8:
+            parts.append(s)
+    return parts
+
+
+def srt_time(sec):
+    ms = int((sec - int(sec)) * 1000)
+    sec = int(sec)
+    return f"{sec//3600:02d}:{(sec%3600)//60:02d}:{sec%60:02d},{ms:03d}"
+
+
+def parse_timecode(t):
+    t = t.replace(',', '.')
+    parts = t.split(':')
+    if len(parts) == 3:
+        return int(parts[0])*3600 + int(parts[1])*60 + float(parts[2])
+    if len(parts) == 2:
+        return int(parts[0])*60 + float(parts[1])
+    return float(parts[0])
+
+
+def convert_vtt_to_scaled_srt(vtt_path, srt_path, speed=1.2):
+    try:
+        txt = open(vtt_path, 'r', encoding='utf-8').read().splitlines()
+        cues = []
+        i = 0
+        while i < len(txt):
+            line = txt[i].strip()
+            if '-->' in line:
+                a, b = [x.strip().split()[0] for x in line.split('-->')[:2]]
+                start = parse_timecode(a) / speed
+                end = parse_timecode(b) / speed
+                i += 1
+                texts = []
+                while i < len(txt) and txt[i].strip():
+                    texts.append(txt[i].strip())
+                    i += 1
+                s = clean(' '.join(texts))
+                if s:
+                    cues.append((start, end, s))
+            i += 1
+        if not cues:
+            return False
+        with open(srt_path, 'w', encoding='utf-8') as f:
+            for idx, (st, en, s) in enumerate(cues, 1):
+                if en <= st:
+                    en = st + 1.2
+                f.write(f"{idx}\n{srt_time(st)} --> {srt_time(en)}\n{s}\n\n")
+        return True
+    except Exception:
+        return False
+
+
+def write_weighted_srt(script, path, total_duration):
+    subs = sentence_split(script)
+    if not subs:
+        subs = [clean(script)[:140] or "VNEWS"]
+    total_chars = max(1, sum(len(x) for x in subs))
+    usable = max(2.0, float(total_duration) - 1.0)
+    cur = 0.5
+    with open(path, "w", encoding="utf-8") as f:
+        for i, s in enumerate(subs, 1):
+            dur = max(1.8, min(7.0, usable * len(s) / total_chars))
+            start = cur
+            end = min(total_duration - 0.15, cur + dur)
+            cur = end + 0.18
+            f.write(f"{i}\n{srt_time(start)} --> {srt_time(end)}\n{s}\n\n")
+            if cur >= total_duration - 0.2:
+                break
+
+
+def tts_script_full(post, emotion):
+    title = clean(post.get("title", ""))
+    text = clean(post.get("text", ""))
+    text = re.sub(r"Nguồn tham khảo:.*", "", text, flags=re.S).strip()
+    prefix = {
+        "urgent": "Tin nhanh.",
+        "warm": "Câu chuyện đáng chú ý.",
+        "serious": "Bản tin nghiêm túc.",
+        "energetic": "Cập nhật nổi bật.",
+    }.get(emotion, "")
+    script = f"{prefix} {title}. {text}".strip()
+    if len(script) > 3600:
+        tmp = script[:3600]
+        cut = max(tmp.rfind("."), tmp.rfind("!"), tmp.rfind("?"))
+        script = tmp[:cut + 1] if cut > 1600 else tmp
+    script = re.sub(r"([\.\!\?])\s*", r"\1\n", script)
+    script = re.sub(r"\n{2,}", "\n", script).strip()
+    return script
+
+
+_PATCH = {('/api/topic_post','POST'),('/api/ai/short/{post_id}','POST'),('/api/ai/short-file/{file_id}','GET'),('/','GET')}
+app.router.routes = [r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
+
+
+@app.post('/api/topic_post')
+async def topic_post_aggregate(request: Request):
+    body = await request.json()
+    topic = base._clean_text(body.get('topic',''))
+    if not topic:
+        return JSONResponse({'error':'missing topic'}, status_code=400)
+    articles = _topic_source_articles(topic, limit=5)
+    if not articles:
+        return JSONResponse({'error':'Không scrape được nội dung bài viết thật cho chủ đề này. Hãy thử chủ đề cụ thể hơn hoặc dán URL trực tiếp.'}, status_code=422)
+    source_blocks = []
+    sources = []
+    image = ""
+    for i, art in enumerate(articles, 1):
+        raw = art.get('raw','')
+        source_blocks.append(f"[Nguồn {i}] {art.get('title','')} ({art.get('via','')})\n{raw[:3000]}")
+        sources.append(art.get('source') or {'title': art.get('title'), 'url': art.get('url'), 'via': art.get('via'), 'excerpt': raw[:600]})
+        if not image and art.get('image'):
+            image = art.get('image')
+    ctx = "\n\n".join(source_blocks)
+    prompt = f"""Bạn là biên tập viên tổng hợp tin tức tiếng Việt.
+
+Chủ đề: {topic}
+
+NHIỆM VỤ:
+- Đọc nội dung của TẤT CẢ các bài nguồn bên dưới.
+- Tổng hợp thành 1 bản tóm tắt chung duy nhất, giống cách tóm tắt qua URL.
+- Không tạo mỗi tiêu đề thành một bài riêng.
+- Không chỉ liệt kê tiêu đề; phải dựa vào nội dung trong từng bài.
+- Không lặp ý giữa các nguồn.
+- Tối đa 6 gạch đầu dòng, mỗi dòng 1 câu rõ ràng.
+- Nếu các nguồn có góc nhìn khác nhau, gộp lại thành ý tổng hợp.
+- Cuối cùng thêm dòng: Nguồn tham khảo: tên website.
+
+Nội dung nguồn:
+{ctx[:16000]}"""
+    text = await prev.base.qwen_generate(prompt, image_url=image or None, max_tokens=1100)
+    text = prev._postprocess_ai_text(text, max_units=7)
+    if 'Nguồn tham khảo:' not in text:
+        text += '\n\n' + prev._source_line(sources)
+    post = base.make_post('Tổng hợp: ' + topic, text, image or base.pollinations_image_url(topic), '', 'topic_aggregate', sources=sources[:5])
+    posts = base._load_ai_wall(); posts.insert(0, post); base._save_ai_wall(posts)
+    return JSONResponse({'post': post, 'count_sources': len(sources)})
+
+
+@app.post('/api/ai/short/{post_id}')
+async def ai_short_full(post_id: str, request: Request):
+    try:
+        body = await request.json()
+    except Exception:
+        body = {}
+    voice = str(body.get('voice','nu')).lower().strip()
+    emotion = str(body.get('emotion','neutral')).lower().strip()
+    speed = max(0.85, min(1.35, float(body.get('speed', 1.2) or 1.2)))
+    posts = base._load_ai_wall()
+    post = next((p for p in posts if str(p.get('id')) == str(post_id)), None)
+    if not post:
+        return JSONResponse({'error':'post not found'}, status_code=404)
+    os.makedirs(base.SHORTS_DIR, exist_ok=True)
+    suffix = f"_{voice}_{emotion}_{str(speed).replace('.', 'p')}_fullv2"
+    out_mp4 = os.path.join(base.SHORTS_DIR, base._safe_name(post_id + suffix) + '.mp4')
+    if os.path.exists(out_mp4):
+        post['video'] = '/api/ai/short-file/' + post_id + suffix
+        base._save_ai_wall(posts)
+        return JSONResponse({'video': post['video'], 'speed': speed, 'subtitles': True})
+    work = os.path.join(base.SHORTS_DIR, base._safe_name(post_id + suffix)); os.makedirs(work, exist_ok=True)
+    img = os.path.join(work,'image.jpg'); frame = os.path.join(work,'frame.jpg'); audio = os.path.join(work,'voice.mp3'); audio_fast=os.path.join(work,'voice_fast.mp3'); srt=os.path.join(work,'subtitles.srt'); vtt=os.path.join(work,'subtitles.vtt')
+    try:
+        base._download_image(post.get('img'), post.get('title','AI news'), img)
+        prev._make_short_frame_full(post, img, frame)
+        script = tts_script_full(post, emotion)
+        edge_voice = {'nam':'vi-VN-NamMinhNeural','male':'vi-VN-NamMinhNeural','nu':'vi-VN-HoaiMyNeural','female':'vi-VN-HoaiMyNeural','mien-nam':'vi-VN-HoaiMyNeural'}.get(voice,'vi-VN-HoaiMyNeural')
+        used_edge = False
+        try:
+            subprocess.run(['python','-m','edge_tts','--voice',edge_voice,'--text',script,'--write-media',audio,'--write-subtitles',vtt], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=260)
+            used_edge = True
+        except Exception:
+            tld = 'com.vn' if voice in ('nu','female','mien-nam') else 'com'
+            try:
+                base.gTTS(script, lang='vi', tld=tld, slow=False).save(audio)
+            except TypeError:
+                base.gTTS(script, lang='vi', slow=False).save(audio)
+        subprocess.run(['ffmpeg','-y','-i',audio,'-filter:a',f'atempo={speed}','-vn',audio_fast], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=220)
+        duration = 45.0
+        try:
+            pr = subprocess.run(['ffprobe','-v','error','-show_entries','format=duration','-of','default=noprint_wrappers=1:no_key=1',audio_fast], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20)
+            duration = float((pr.stdout or b'45').decode().strip() or 45)
+        except Exception:
+            pass
+        if used_edge and os.path.exists(vtt):
+            ok = convert_vtt_to_scaled_srt(vtt, srt, speed=speed)
+            if not ok:
+                write_weighted_srt(script, srt, duration)
+        else:
+            write_weighted_srt(script, srt, duration)
+        vf = "scale=1080:1920,subtitles='{}':force_style='FontName=DejaVu Sans,FontSize=16,PrimaryColour=&H00FFFFFF,OutlineColour=&HAA000000,BorderStyle=1,Outline=1.5,Shadow=0,Alignment=2,MarginV=42'".format(srt.replace("'", "\\'"))
+        cmd = ['ffmpeg','-y','-loop','1','-i',frame,'-i',audio_fast,'-shortest','-c:v','libx264','-tune','stillimage','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k','-vf',vf,out_mp4]
+        subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=420)
+        post['video'] = '/api/ai/short-file/' + post_id + suffix
+        post['short_voice'] = voice; post['short_emotion'] = emotion; post['short_speed'] = speed; post['short_subtitles'] = True
+        base._save_ai_wall(posts)
+        return JSONResponse({'video': post['video'], 'voice': voice, 'emotion': emotion, 'speed': speed, 'subtitles': True, 'duration': duration})
+    except Exception as e:
+        return JSONResponse({'error':'Không tạo được shorts: '+str(e)[:180]}, status_code=500)
+
+
+@app.get('/api/ai/short-file/{file_id}')
+def ai_short_file_full(file_id: str):
+    path = os.path.join(base.SHORTS_DIR, base._safe_name(file_id) + '.mp4')
+    if not os.path.exists(path):
+        return JSONResponse({'error':'not found'}, status_code=404)
+    return FileResponse(path, media_type='video/mp4', filename=f'vnews-ai-{file_id}.mp4')
+
+
+app.router.routes = [r for r in app.router.routes if not (getattr(r,'path',None)=='/' and 'GET' in getattr(r,'methods',set()))]
+
+@app.get('/')
+async def index_fix2():
+    with open('/app/static/index.html','r',encoding='utf-8') as f:
+        html = f.read()
+    inject = prev.PATCH_INJECT + r'''
+<script>
+(function(){
+window.createTopicPost=function(){let inp=document.getElementById('ai-topic-input');let topic=(inp&&inp.value||'').trim();if(!topic)return alert('Nhập chủ đề trước');fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic})}).then(r=>r.json().then(j=>({ok:r.ok,j}))).then(({ok,j})=>{if(ok&&j.post){window.location.reload();alert('Đã tổng hợp NỘI DUNG các bài nguồn thành 1 bản tóm tắt trên Tường AI');}else alert(j.error||'Lỗi tạo bài')}).catch(e=>alert(e.message||'Lỗi tạo bài'));};
+})();
+</script>
+'''
+    return HTMLResponse(html.replace('</body>', inject+'\n</body>'))