Restore: ai_fix2.py (full), app_entry.py, patch_extra.py, restore_runner.py, app_clean.py, wc2026_scraper.py from c93b544

ai_fix2.py

@@ -0,0 +1,207 @@
+import os, re, subprocess, html as html_lib, json
+from urllib.parse import quote_plus, urlparse, parse_qs, unquote
+import requests
+import ai_patch as prev
+from ai_patch import app
+from fastapi import Request
+from fastapi.responses import JSONResponse, HTMLResponse, FileResponse
+
+base = prev.base
+
+
+def clean(s):
+    return re.sub(r"\s+", " ", html_lib.unescape(s or "")).strip()
+
+
+def _is_real_article_text(raw):
+    raw = clean(raw)
+    if len(raw) < 500:
+        return False
+    sentences = re.split(r"(?<=[\.\!\?])\s+", raw)
+    long_sentences = [s for s in sentences if len(s) > 45]
+    return len(long_sentences) >= 5
+
+
+def _extract_ddg_url(href):
+    if not href:
+        return ""
+    if href.startswith("//"):
+        href = "https:" + href
+    if "duckduckgo.com/l/" in href:
+        try:
+            qs = parse_qs(urlparse(href).query)
+            if qs.get("uddg"):
+                return unquote(qs["uddg"][0])
+        except Exception:
+            pass
+    return href
+
+
+def _ddg_article_urls(topic, limit=12):
+    urls = []
+    try:
+        q = quote_plus(topic + " tin tức bài viết phân tích")
+        r = requests.get("https://html.duckduckgo.com/html/?q=" + q, headers=base.HEADERS, timeout=18)
+        r.encoding = "utf-8"
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(r.text, "lxml")
+        for a in soup.select("a.result__a"):
+            u = _extract_ddg_url(a.get("href", ""))
+            if not u.startswith("http"):
+                continue
+            if any(bad in u for bad in ["google.com", "youtube.com", "facebook.com", "x.com", "twitter.com"]):
+                continue
+            if u not in urls:
+                urls.append(u)
+            if len(urls) >= limit:
+                break
+    except Exception:
+        pass
+    return urls
+
+
+def _rss_article_urls(topic, limit=10):
+    out = []
+    try:
+        url = "https://news.google.com/rss/search?q=" + quote_plus(topic) + "&hl=vi&gl=VN&ceid=VN:vi"
+        r = requests.get(url, headers=base.HEADERS, timeout=15)
+        r.encoding = "utf-8"
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(r.text, "xml")
+        for it in soup.find_all("item")[:limit]:
+            title = it.find("title").get_text(" ", strip=True) if it.find("title") else ""
+            link = it.find("link").get_text(strip=True) if it.find("link") else ""
+            src = it.find("source").get_text(" ", strip=True) if it.find("source") else base._domain(link)
+            if title and link:
+                out.append({"title": title, "url": link, "via": src, "excerpt": title})
+    except Exception:
+        pass
+    return out
+
+
+def _topic_source_articles(topic, limit=5):
+    """Scrape actual article bodies. Do not accept title-only sources."""
+    candidates = []
+    seen = set()
+
+    for u in _ddg_article_urls(topic, limit=14):
+        if u not in seen:
+            seen.add(u)
+            candidates.append({"url": u, "title": "", "via": base._domain(u)})
+
+    try:
+        _ctx, srcs = base.web_context(topic, limit=8)
+        for s in srcs or []:
+            u = s.get("url") or ""
+            if u.startswith("http") and u not in seen:
+                seen.add(u)
+                candidates.append(s)
+    except Exception:
+        pass
+
+    for s in _rss_article_urls(topic, limit=10):
+        u = s.get("url") or ""
+        if u.startswith("http") and u not in seen:
+            seen.add(u)
+            candidates.append(s)
+
+    out = []
+    for s in candidates[:24]:
+        url = s.get("url") or ""
+        try:
+            page = base.scrape_any_url(url)
+            raw = (page.get("summary", "") + "\n" + page.get("text", "")).strip()
+            if not _is_real_article_text(raw):
+                continue
+            title = page.get("title") or s.get("title") or url
+            via = page.get("via") or s.get("via") or base._domain(url)
+            out.append({
+                "title": title,
+                "url": url,
+                "raw": raw,
+                "image": page.get("image") or "",
+                "via": via,
+                "source": {"title": title, "url": url, "excerpt": raw[:700], "via": via}
+            })
+            if len(out) >= limit:
+                break
+        except Exception:
+            continue
+    return out[:limit]
+
+
+def sentence_split(text):
+    text = re.sub(r"^[•\-\*]\s*", "", text or "", flags=re.M)
+    text = re.sub(r"\n+", ". ", text)
+    parts = []
+    for s in re.split(r"(?<=[\.\!\?])\s+", text):
+        s = clean(s)
+        if len(s) >= 8:
+            parts.append(s)
+    return parts
+
+
+def srt_time(sec):
+    ms = int((sec - int(sec)) * 1000)
+    sec = int(sec)
+    return f"{sec//3600:02d}:{(sec%3600)//60:02d}:{sec%60:02d},{ms:03d}"
+
+
+def parse_timecode(t):
+    t = t.replace(',', '.')
+    parts = t.split(':')
+    if len(parts) == 3:
+        return int(parts[0])*3600 + int(parts[1])*60 + float(parts[2])
+    if len(parts) == 2:
+        return int(parts[0])*60 + float(parts[1])
+    return float(parts[0])
+
+
+def convert_vtt_to_scaled_srt(vtt_path, srt_path, speed=1.2):
+    try:
+        txt = open(vtt_path, 'r', encoding='utf-8').read().splitlines()
+        cues = []
+        i = 0
+        while i < len(txt):
+            line = txt[i].strip()
+            if '-->' in line:
+                a, b = [x.strip().split()[0] for x in line.split('-->')[:2]]
+                start = parse_timecode(a) / speed
+                end = parse_timecode(b) / speed
+                i += 1
+                texts = []
+                while i < len(txt) and txt[i].strip():
+                    texts.append(txt[i].strip())
+                    i += 1
+                s = clean(' '.join(texts))
+                if s:
+                    cues.append((start, end, s))
+            i += 1
+        if not cues:
+            return False
+        with open(srt_path, 'w', encoding='utf-8') as f:
+            for idx, (st, en, s) in enumerate(cues, 1):
+                if en <= st:
+                    en = st + 1.2
+                f.write(f"{idx}\n{srt_time(st)} --> {srt_time(en)}\n{s}\n\n")
+        return True
+    except Exception:
+        return False
+
+
+def write_weighted_srt(script, path, total_duration):
+    subs = sentence_split(script)
+    if not subs:
+        subs = [clean(script)[:140] or "VNEWS"]
+    total_chars = max(1, sum(len(x) for x in subs))
+    usable = max(2.0, float(total_duration) - 1.0)
+    cur = 0.5
+    with open(path, "w", encoding="utf-8") as f:
+        for i, s in enumerate(subs, 1):
+            dur = max(1.8, min(7.0, usable * len(s) / total_chars))
+            start = cur
+            end = min(total_duration - 0.15, cur + dur)
+            cur = end + 0.18
+            f.write(f"{i}\n{srt_time(start)} --> {srt_time(end)}\n{s}\n\n")
+            if cur >= total_duration - 0.2:
+                break