import os, re, json, time, random, html as html_lib, subprocess, uuid
from urllib.parse import quote_plus, quote, urlparse
from typing import Optional
import requests
from bs4 import BeautifulSoup
from fastapi import Request
from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
from main import app
try:
from huggingface_hub import AsyncInferenceClient
except Exception:
AsyncInferenceClient = None
try:
from gtts import gTTS
except Exception:
gTTS = None
try:
from PIL import Image, ImageDraw, ImageFont
except Exception:
Image = ImageDraw = ImageFont = None
def _hf_token():
for k in ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_API_TOKEN"):
v = os.getenv(k, "").strip()
if v:
return v
return ""
HF_TOKEN = _hf_token()
QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
WALL_FILE = os.path.join(DATA_DIR, "ai_wall_posts.json")
SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts")
HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"vi-VN,vi;q=0.9,en;q=0.8"}
LAST_QWEN_ERROR = ""
def _load_ai_wall():
try:
if os.path.exists(WALL_FILE):
with open(WALL_FILE, "r", encoding="utf-8") as f:
return json.load(f)
except Exception:
pass
return []
def _save_ai_wall(posts):
try:
os.makedirs(os.path.dirname(WALL_FILE), exist_ok=True)
tmp = WALL_FILE + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(posts[:100], f, ensure_ascii=False)
os.replace(tmp, WALL_FILE)
except Exception:
pass
def _clean_text(s: str) -> str:
s = html_lib.unescape(s or "")
return re.sub(r"\s+", " ", s).strip()
def _domain(u):
try:
return urlparse(u).netloc.replace("www.", "")
except Exception:
return ""
def _reader_url(target_url: str) -> str:
safe = quote(target_url, safe=":/?#[]@!$&'()*+,;=%")
return "https://r.jina.ai/http://" + safe
def jina_reader_markdown(url: str) -> str:
jr = _reader_url(url)
r = requests.get(jr, headers={"Accept":"text/markdown,text/plain,*/*", "X-Return-Format":"markdown", "User-Agent":"Mozilla/5.0"}, timeout=35)
r.raise_for_status()
return r.text or ""
def _parse_jina_markdown(md: str, url: str):
lines = [x.rstrip() for x in (md or "").splitlines()]
title = ""; image = ""; content_lines = []; in_content = False
for ln in lines:
if ln.startswith("Title:") and not title:
title = _clean_text(ln.replace("Title:", "", 1)); continue
if ln.startswith("URL Source:"):
continue
if ln.startswith("Markdown Content:"):
in_content = True; continue
mimg = re.search(r'!\[[^\]]*\]\((https?://[^)]+)\)', ln)
if mimg and not image:
image = mimg.group(1)
if in_content or (title and not ln.startswith("Title:")):
if ln.strip(): content_lines.append(ln)
text = "\n".join(content_lines)
text = re.sub(r'!\[[^\]]*\]\([^)]+\)', '', text)
text = re.sub(r'\[[^\]]+\]\((https?://[^)]+)\)', lambda m: m.group(0).split('](')[0].strip('['), text)
paras = []
for part in re.split(r'\n{2,}|\n(?=#{1,3}\s)', text):
t = _clean_text(re.sub(r'^#{1,6}\s*', '', part))
if len(t) >= 40: paras.append(t)
if len(paras) >= 35: break
if not title and paras: title = paras[0][:90]
return {"url": url, "title": title or url, "summary": paras[0] if paras else "", "text": "\n".join(paras), "image": image, "images": [image] if image else [], "via":"jina"}
def _best_content_block(soup):
best, best_score = None, 0
for el in soup.find_all(["article", "main", "section", "div"]):
ps = el.find_all("p")
txt = " ".join(p.get_text(" ", strip=True) for p in ps)
score = len(ps) * 100 + len(txt)
cls = " ".join(el.get("class", []))
if any(k in cls.lower() for k in ["content", "article", "detail", "body", "post", "entry"]): score += 800
if score > best_score: best, best_score = el, score
return best
def scrape_any_url_direct(url: str):
r = requests.get(url, headers=HEADERS, timeout=18)
if r.status_code in {401,403,406,409,429,451,503}: raise RuntimeError(f"blocked status {r.status_code}")
r.encoding = "utf-8"; soup = BeautifulSoup(r.text, "lxml")
for tag in soup.find_all(["script","style","nav","footer","aside","form","noscript"]): tag.decompose()
title = soup.find("h1").get_text(" ", strip=True) if soup.find("h1") else ""
if not title:
ogt = soup.find("meta", property="og:title") or soup.find("meta", attrs={"name":"title"})
title = ogt.get("content", "") if ogt else (soup.title.get_text(strip=True) if soup.title else "")
desc_tag = soup.find("meta", property="og:description") or soup.find("meta", attrs={"name":"description"})
summary = desc_tag.get("content", "") if desc_tag else ""
img_tag = soup.find("meta", property="og:image") or soup.find("meta", attrs={"name":"twitter:image"})
image = img_tag.get("content", "") if img_tag else ""
if image and image.startswith("//"): image = "https:" + image
block = _best_content_block(soup) or soup
paras, images, seen_p, seen_i = [], [], set(), set()
for p in block.find_all("p"):
t = _clean_text(p.get_text(" ", strip=True))
if len(t) >= 40 and t not in seen_p:
seen_p.add(t); paras.append(t)
if len(paras) >= 35: break
for im in block.find_all("img"):
src = im.get("data-src") or im.get("data-original") or im.get("src") or ""
if src.startswith("//"): src = "https:" + src
if src and "base64" not in src and src not in seen_i:
seen_i.add(src); images.append(src)
if len(images) >= 8: break
if not image and images: image = images[0]
data = {"url":url,"title":_clean_text(title),"summary":_clean_text(summary),"text":"\n".join(paras),"image":image,"images":images,"via":"direct"}
if len(data["text"]) < 160: raise RuntimeError("direct scrape too short")
return data
def scrape_any_url(url: str):
try:
return scrape_any_url_direct(url)
except Exception as direct_error:
md = jina_reader_markdown(url)
data = _parse_jina_markdown(md, url)
if len(data.get("text", "")) < 120:
raise RuntimeError(f"Jina Reader also returned too little content; direct={direct_error}")
return data
def pollinations_image_url(topic: str, width=1024, height=576):
prompt = f"editorial illustration, topic: {topic}, modern clean cinematic news image, high quality, no text, no watermark"
return f"https://image.pollinations.ai/prompt/{quote_plus(prompt)}?width={width}&height={height}&seed={random.randint(1,999999)}&nologo=true&private=true"
def jina_search(query: str, limit=6):
for accept in ("application/json", "text/plain"):
try:
url = "https://s.jina.ai/" + quote(query, safe="")
r = requests.get(url, headers={"Accept":accept, "User-Agent":"Mozilla/5.0"}, timeout=30)
if accept == "application/json" and r.status_code == 200 and r.text.strip().startswith("{"):
payload = r.json(); data = payload.get("data", [])
if isinstance(data, dict): data=[data]
out=[]
for item in data[:limit]:
out.append({"title":_clean_text(item.get("title","")),"url":item.get("url",""),"description":_clean_text(item.get("description","")),"content":_clean_text(item.get("content",""))[:1200]})
if out: return out
elif r.status_code == 200 and r.text.strip():
chunks = re.split(r'\n(?=\[\d+\]|Title:)', r.text)
out=[]
for c in chunks[:limit]:
title = re.search(r'Title:\s*(.+)', c)
urlm = re.search(r'URL Source:\s*(https?://\S+)', c)
desc = re.search(r'Description:\s*(.+)', c)
out.append({"title":_clean_text(title.group(1) if title else c[:80]),"url":urlm.group(1) if urlm else "","description":_clean_text(desc.group(1) if desc else ""),"content":_clean_text(c)[:1200]})
if out: return out
except Exception:
pass
return []
def duck_context(topic: str, limit=6):
try:
url = "https://duckduckgo.com/html/?q=" + quote_plus(topic + " tin tức phân tích bối cảnh")
r = requests.get(url, headers=HEADERS, timeout=12); soup = BeautifulSoup(r.text, "lxml")
results=[]
for res in soup.select(".result")[:limit]:
a = res.select_one(".result__a") or res.find("a", href=True)
href = a.get("href", "") if a else ""; title = _clean_text(a.get_text(" ", strip=True) if a else "")
sn = res.select_one(".result__snippet"); snippet = _clean_text(sn.get_text(" ", strip=True) if sn else "")
if title or snippet: results.append({"title":title,"url":href,"description":snippet,"content":snippet})
return results
except Exception: return []
def web_context(topic: str, limit=6):
queries = [topic + " tin tức phân tích bối cảnh", topic]
results=[]
for q in queries:
results = jina_search(q, limit=limit)
if results: break
if not results: results = duck_context(topic, limit=limit)
enriched=[]
for item in results[:limit]:
content = item.get("content") or item.get("description") or ""
if len(content) < 250 and item.get("url", "").startswith("http"):
try:
page = scrape_any_url(item["url"])
content = (page.get("summary") or "") + " " + page.get("text", "")[:900]
except Exception: pass
enriched.append({**item, "excerpt": _clean_text(content)[:1200]})
context = "\n".join([f"- {x.get('title') or _domain(x.get('url',''))} ({_domain(x.get('url',''))}): {x.get('excerpt') or x.get('description','')}" for x in enriched if x.get("excerpt") or x.get("title")])
return context, enriched
async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 1200):
global LAST_QWEN_ERROR, HF_TOKEN
HF_TOKEN = _hf_token()
if not HF_TOKEN:
LAST_QWEN_ERROR = "Không tìm thấy token trong env: HF_TOKEN / HUGGINGFACEHUB_API_TOKEN / HUGGING_FACE_HUB_TOKEN. Hãy Restart Space sau khi thêm secret."
return None
if not AsyncInferenceClient:
LAST_QWEN_ERROR = "Thiếu thư viện huggingface_hub hoặc import lỗi."
return None
errors=[]; models=[]
for m in [QWEN_VL_MODEL, "Qwen/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]:
if m and m not in models: models.append(m)
for model in models:
try:
client = AsyncInferenceClient(provider="auto", api_key=HF_TOKEN, timeout=90)
content=[]
if image_url: content.append({"type":"image_url", "image_url":{"url": image_url}})
content.append({"type":"text", "text": prompt})
messages=[{"role":"system","content":"Bạn là biên tập viên AI tiếng Việt. Nhiệm vụ chính là tóm tắt súc tích nội dung nguồn, không viết lại toàn bài, không lặp ý, không bịa chi tiết."},{"role":"user","content":content}]
resp=await client.chat_completion(model=model, messages=messages, max_tokens=max_tokens, temperature=0.45, top_p=0.85)
txt=(resp.choices[0].message.content or "").strip()
if txt:
LAST_QWEN_ERROR=""; return txt
except Exception as e:
errors.append(f"{model}: {type(e).__name__}: {str(e)[:220]}")
LAST_QWEN_ERROR = " | ".join(errors) or "Qwen không trả nội dung."
print("[qwen errors]", LAST_QWEN_ERROR)
return None
def make_post(title, text, image, source_url, kind, sources=None):
return {"id":str(int(time.time()*1000))+str(random.randint(100,999)),"title":title,"text":text,"img":image,"url":source_url,"kind":kind,"sources":sources or [],"video":"","ts":int(time.time())}
@app.get("/api/ai_wall")
def api_ai_wall(): return JSONResponse({"posts":_load_ai_wall()[:80]})
@app.get("/api/ai/status")
def api_ai_status():
return JSONResponse({"has_token":bool(_hf_token()),"client_imported":AsyncInferenceClient is not None,"model":QWEN_VL_MODEL,"last_error":LAST_QWEN_ERROR,"tts_ready":gTTS is not None})
@app.post("/api/ai/topic")
async def api_ai_topic(request:Request):
body=await request.json(); topic=_clean_text(body.get("topic",""))
if not topic: return JSONResponse({"error":"missing topic"},status_code=400)
ctx,sources=web_context(topic)
if not ctx: return JSONResponse({"error":"Không lấy được dữ liệu internet cho chủ đề này. Hãy thử chủ đề cụ thể hơn."},status_code=422)
image=pollinations_image_url(topic)
prompt=f"""Tóm tắt chủ đề để đăng Tường AI.
Chủ đề: {topic}
Yêu cầu:
- Chỉ tóm tắt các ý chính dựa trên nguồn internet bên dưới.
- Không viết lại thành bài dài, không lan man, không lặp lại.
- 1 tiêu đề ngắn.
- 3-5 gạch đầu dòng, mỗi dòng tối đa 1 câu.
- 1 câu kết luận ngắn.
- Cuối cùng ghi nguồn tham khảo bằng tên website.
Nguồn/bối cảnh internet:
{ctx}"""
text=await qwen_generate(prompt,image_url=image,max_tokens=900)
if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503)
post=make_post(topic,text,image,"","topic",sources=sources[:5]); posts=_load_ai_wall(); posts.insert(0,post); _save_ai_wall(posts)
return JSONResponse({"post":post})
@app.post("/api/ai/url")
async def api_ai_url(request:Request):
body=await request.json(); url=_clean_text(body.get("url",""))
if not url.startswith("http"): return JSONResponse({"error":"missing url"},status_code=400)
try: data=scrape_any_url(url)
except Exception as e: return JSONResponse({"error":"Không scrape được URL: "+str(e)[:180]},status_code=422)
raw=(data.get("summary","")+"\n"+data.get("text","")).strip()
if len(raw)<120: return JSONResponse({"error":"URL không có đủ nội dung để Qwen tóm tắt"},status_code=422)
prompt=f"""Tóm tắt bài viết nguồn dưới đây để đăng lên Tường AI VNEWS.
Yêu cầu bắt buộc:
- Chỉ viết TÓM TẮT theo nội dung chính, không viết lại toàn bộ bài.
- Ngắn gọn, xúc tích, cụ thể.
- Tránh lặp lại ý và câu chữ trong bài gốc.
- Không thêm chi tiết ngoài nguồn.
- Độ dài tối đa 6 gạch đầu dòng hoặc 2 đoạn ngắn.
- Nếu có nhân vật/sự kiện/thời điểm quan trọng, giữ lại.
Tiêu đề gốc: {data.get('title','')}
Nguồn scrape: {data.get('via','')}
Nội dung gốc:
{raw[:16000]}"""
text=await qwen_generate(prompt,image_url=data.get("image") or None,max_tokens=900)
if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503)
post=make_post(data.get("title") or "Bài viết",text,data.get("image") or "",url,"url",sources=[{"title":data.get("title"),"url":url,"excerpt":raw[:500],"via":data.get("via")}])
posts=_load_ai_wall(); posts.insert(0,post); _save_ai_wall(posts)
return JSONResponse({"post":post})
@app.post("/api/rewrite_share")
async def api_rewrite_share(request:Request):
body=await request.json(); url=_clean_text(body.get("url",""))
if not url.startswith("http"): return JSONResponse({"error":"missing url"},status_code=400)
try: data=scrape_any_url(url)
except Exception as e: return JSONResponse({"error":"Không đọc được bài viết: "+str(e)[:180]},status_code=422)
raw=(data.get("summary","")+"\n"+data.get("text","")).strip()
prompt=f"""Tóm tắt bài viết sau bằng tiếng Việt.
Yêu cầu:
- Chỉ nêu nội dung chính, không viết lại toàn bài.
- Ngắn gọn, cụ thể, dễ hiểu.
- Tối đa 5 gạch đầu dòng hoặc 2 đoạn ngắn.
- Không lặp ý, không thêm chi tiết ngoài nguồn.
Tiêu đề: {data.get('title','')}
Nội dung:
{raw[:16000]}"""
text=await qwen_generate(prompt,image_url=data.get("image") or None,max_tokens=900)
if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503)
post=make_post(data.get("title") or "Bài viết",text,data.get("image") or "",url,"rewrite")
posts=_load_ai_wall(); posts.insert(0,post); _save_ai_wall(posts)
return JSONResponse({"post":post})
def _safe_name(s): return re.sub(r"[^a-zA-Z0-9_-]+", "_", s)[:80]
def _wrap_text(text, width=32, max_lines=10):
words=_clean_text(text).split(); lines=[]; cur=""
for w in words:
if len(cur)+len(w)+1<=width: cur=(cur+" "+w).strip()
else:
if cur: lines.append(cur)
cur=w
if len(lines)>=max_lines: break
if cur and len(lines)target_ratio:
new_h=target[1]; new_w=int(new_h*im_ratio)
else:
new_w=target[0]; new_h=int(new_w/im_ratio)
im=im.resize((new_w,new_h))
left=(new_w-target[0])//2; top=(new_h-target[1])//2
im=im.crop((left,top,left+target[0],top+target[1]))
bg.paste(im,(0,0))
except Exception:
pass
draw=ImageDraw.Draw(bg)
try:
font_title=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",52)
font_body=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",40)
font_label=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",32)
except Exception:
font_title=font_body=font_label=None
draw.rectangle((0,780,W,H),fill=(14,14,14))
draw.text((54,830),"VNEWS · Tường AI",fill=(92,184,122),font=font_label)
title=_wrap_text(post.get("title",""),24,3)
draw.multiline_text((54,900),title,fill=(255,255,255),font=font_title,spacing=10)
body=_wrap_text(post.get("text",""),34,10)
draw.multiline_text((54,1120),body,fill=(220,220,220),font=font_body,spacing=12)
bg.save(out_path,quality=92)
def _download_image(url, fallback_topic, out_path):
if url:
try:
r=requests.get(url,headers=HEADERS,timeout=15)
if r.status_code==200 and len(r.content)>1000:
with open(out_path,"wb") as f:f.write(r.content)
return out_path
except Exception: pass
# fallback generated image
gen=pollinations_image_url(fallback_topic)
try:
r=requests.get(gen,headers=HEADERS,timeout=25)
if r.status_code==200 and len(r.content)>1000:
with open(out_path,"wb") as f:f.write(r.content)
return out_path
except Exception: pass
if Image:
Image.new("RGB",(1080,860),(30,55,42)).save(out_path)
return out_path
raise RuntimeError("Không tạo được ảnh")
def _short_script(post):
txt=_clean_text(post.get("text",""))
# TTS should read concise summary, not entire huge post.
if len(txt)>900: txt=txt[:900].rsplit(" ",1)[0]+"."
return f"{post.get('title','')}. {txt}"
@app.post("/api/ai/short/{post_id}")
def api_ai_short(post_id:str):
posts=_load_ai_wall(); post=next((p for p in posts if str(p.get("id"))==str(post_id)),None)
if not post: return JSONResponse({"error":"post not found"},status_code=404)
os.makedirs(SHORTS_DIR,exist_ok=True)
out_mp4=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4")
if os.path.exists(out_mp4):
post["video"]="/api/ai/short-file/"+post_id
_save_ai_wall(posts)
return JSONResponse({"video":post["video"]})
if gTTS is None: return JSONResponse({"error":"gTTS chưa sẵn sàng"},status_code=503)
work=os.path.join(SHORTS_DIR,_safe_name(post_id))
os.makedirs(work,exist_ok=True)
img=os.path.join(work,"image.jpg"); frame=os.path.join(work,"frame.jpg"); audio=os.path.join(work,"voice.mp3")
try:
_download_image(post.get("img"), post.get("title","AI news"), img)
_make_short_frame(post,img,frame)
script=_short_script(post)
gTTS(script,lang="vi").save(audio)
cmd=["ffmpeg","-y","-loop","1","-i",frame,"-i",audio,"-shortest","-c:v","libx264","-tune","stillimage","-pix_fmt","yuv420p","-c:a","aac","-b:a","128k","-vf","scale=1080:1920",out_mp4]
subprocess.run(cmd,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=180)
post["video"]="/api/ai/short-file/"+post_id
_save_ai_wall(posts)
return JSONResponse({"video":post["video"]})
except Exception as e:
return JSONResponse({"error":"Không tạo được shorts: "+str(e)[:180]},status_code=500)
@app.get("/api/ai/short-file/{post_id}")
def api_ai_short_file(post_id:str):
path=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4")
if not os.path.exists(path): return JSONResponse({"error":"not found"},status_code=404)
return FileResponse(path,media_type="video/mp4",filename=f"vnews-ai-{post_id}.mp4")
# Override index route to show Tường AI slide + shorts generation.
app.router.routes=[r for r in app.router.routes if not (getattr(r,'path',None)=='/' and 'GET' in getattr(r,'methods',set()))]
AI_INJECT=r'''
'''
@app.get('/')
async def index_ai():
with open('/app/static/index.html','r',encoding='utf-8') as f: html=f.read()
return HTMLResponse(html.replace('