import os, re, json, time, random, html as html_lib, subprocess, uuid from urllib.parse import quote_plus, quote, urlparse from typing import Optional import requests from bs4 import BeautifulSoup from fastapi import Request from fastapi.responses import HTMLResponse, JSONResponse, FileResponse from main import app try: from huggingface_hub import AsyncInferenceClient except Exception: AsyncInferenceClient = None try: from gtts import gTTS except Exception: gTTS = None try: from PIL import Image, ImageDraw, ImageFont except Exception: Image = ImageDraw = ImageFont = None def _hf_token(): for k in ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_API_TOKEN"): v = os.getenv(k, "").strip() if v: return v return "" HF_TOKEN = _hf_token() QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct") DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data" WALL_FILE = os.path.join(DATA_DIR, "ai_wall_posts.json") SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts") HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"vi-VN,vi;q=0.9,en;q=0.8"} LAST_QWEN_ERROR = "" def _load_ai_wall(): try: if os.path.exists(WALL_FILE): with open(WALL_FILE, "r", encoding="utf-8") as f: return json.load(f) except Exception: pass return [] def _save_ai_wall(posts): try: os.makedirs(os.path.dirname(WALL_FILE), exist_ok=True) tmp = WALL_FILE + ".tmp" with open(tmp, "w", encoding="utf-8") as f: json.dump(posts[:100], f, ensure_ascii=False) os.replace(tmp, WALL_FILE) except Exception: pass def _clean_text(s: str) -> str: s = html_lib.unescape(s or "") return re.sub(r"\s+", " ", s).strip() def _domain(u): try: return urlparse(u).netloc.replace("www.", "") except Exception: return "" def _reader_url(target_url: str) -> str: safe = quote(target_url, safe=":/?#[]@!$&'()*+,;=%") return "https://r.jina.ai/http://" + safe def jina_reader_markdown(url: str) -> str: jr = _reader_url(url) r = requests.get(jr, headers={"Accept":"text/markdown,text/plain,*/*", "X-Return-Format":"markdown", "User-Agent":"Mozilla/5.0"}, timeout=35) r.raise_for_status() return r.text or "" def _parse_jina_markdown(md: str, url: str): lines = [x.rstrip() for x in (md or "").splitlines()] title = ""; image = ""; content_lines = []; in_content = False for ln in lines: if ln.startswith("Title:") and not title: title = _clean_text(ln.replace("Title:", "", 1)); continue if ln.startswith("URL Source:"): continue if ln.startswith("Markdown Content:"): in_content = True; continue mimg = re.search(r'!\[[^\]]*\]\((https?://[^)]+)\)', ln) if mimg and not image: image = mimg.group(1) if in_content or (title and not ln.startswith("Title:")): if ln.strip(): content_lines.append(ln) text = "\n".join(content_lines) text = re.sub(r'!\[[^\]]*\]\([^)]+\)', '', text) text = re.sub(r'\[[^\]]+\]\((https?://[^)]+)\)', lambda m: m.group(0).split('](')[0].strip('['), text) paras = [] for part in re.split(r'\n{2,}|\n(?=#{1,3}\s)', text): t = _clean_text(re.sub(r'^#{1,6}\s*', '', part)) if len(t) >= 40: paras.append(t) if len(paras) >= 35: break if not title and paras: title = paras[0][:90] return {"url": url, "title": title or url, "summary": paras[0] if paras else "", "text": "\n".join(paras), "image": image, "images": [image] if image else [], "via":"jina"} def _best_content_block(soup): best, best_score = None, 0 for el in soup.find_all(["article", "main", "section", "div"]): ps = el.find_all("p") txt = " ".join(p.get_text(" ", strip=True) for p in ps) score = len(ps) * 100 + len(txt) cls = " ".join(el.get("class", [])) if any(k in cls.lower() for k in ["content", "article", "detail", "body", "post", "entry"]): score += 800 if score > best_score: best, best_score = el, score return best def scrape_any_url_direct(url: str): r = requests.get(url, headers=HEADERS, timeout=18) if r.status_code in {401,403,406,409,429,451,503}: raise RuntimeError(f"blocked status {r.status_code}") r.encoding = "utf-8"; soup = BeautifulSoup(r.text, "lxml") for tag in soup.find_all(["script","style","nav","footer","aside","form","noscript"]): tag.decompose() title = soup.find("h1").get_text(" ", strip=True) if soup.find("h1") else "" if not title: ogt = soup.find("meta", property="og:title") or soup.find("meta", attrs={"name":"title"}) title = ogt.get("content", "") if ogt else (soup.title.get_text(strip=True) if soup.title else "") desc_tag = soup.find("meta", property="og:description") or soup.find("meta", attrs={"name":"description"}) summary = desc_tag.get("content", "") if desc_tag else "" img_tag = soup.find("meta", property="og:image") or soup.find("meta", attrs={"name":"twitter:image"}) image = img_tag.get("content", "") if img_tag else "" if image and image.startswith("//"): image = "https:" + image block = _best_content_block(soup) or soup paras, images, seen_p, seen_i = [], [], set(), set() for p in block.find_all("p"): t = _clean_text(p.get_text(" ", strip=True)) if len(t) >= 40 and t not in seen_p: seen_p.add(t); paras.append(t) if len(paras) >= 35: break for im in block.find_all("img"): src = im.get("data-src") or im.get("data-original") or im.get("src") or "" if src.startswith("//"): src = "https:" + src if src and "base64" not in src and src not in seen_i: seen_i.add(src); images.append(src) if len(images) >= 8: break if not image and images: image = images[0] data = {"url":url,"title":_clean_text(title),"summary":_clean_text(summary),"text":"\n".join(paras),"image":image,"images":images,"via":"direct"} if len(data["text"]) < 160: raise RuntimeError("direct scrape too short") return data def scrape_any_url(url: str): try: return scrape_any_url_direct(url) except Exception as direct_error: md = jina_reader_markdown(url) data = _parse_jina_markdown(md, url) if len(data.get("text", "")) < 120: raise RuntimeError(f"Jina Reader also returned too little content; direct={direct_error}") return data def pollinations_image_url(topic: str, width=1024, height=576): prompt = f"editorial illustration, topic: {topic}, modern clean cinematic news image, high quality, no text, no watermark" return f"https://image.pollinations.ai/prompt/{quote_plus(prompt)}?width={width}&height={height}&seed={random.randint(1,999999)}&nologo=true&private=true" def jina_search(query: str, limit=6): for accept in ("application/json", "text/plain"): try: url = "https://s.jina.ai/" + quote(query, safe="") r = requests.get(url, headers={"Accept":accept, "User-Agent":"Mozilla/5.0"}, timeout=30) if accept == "application/json" and r.status_code == 200 and r.text.strip().startswith("{"): payload = r.json(); data = payload.get("data", []) if isinstance(data, dict): data=[data] out=[] for item in data[:limit]: out.append({"title":_clean_text(item.get("title","")),"url":item.get("url",""),"description":_clean_text(item.get("description","")),"content":_clean_text(item.get("content",""))[:1200]}) if out: return out elif r.status_code == 200 and r.text.strip(): chunks = re.split(r'\n(?=\[\d+\]|Title:)', r.text) out=[] for c in chunks[:limit]: title = re.search(r'Title:\s*(.+)', c) urlm = re.search(r'URL Source:\s*(https?://\S+)', c) desc = re.search(r'Description:\s*(.+)', c) out.append({"title":_clean_text(title.group(1) if title else c[:80]),"url":urlm.group(1) if urlm else "","description":_clean_text(desc.group(1) if desc else ""),"content":_clean_text(c)[:1200]}) if out: return out except Exception: pass return [] def duck_context(topic: str, limit=6): try: url = "https://duckduckgo.com/html/?q=" + quote_plus(topic + " tin tức phân tích bối cảnh") r = requests.get(url, headers=HEADERS, timeout=12); soup = BeautifulSoup(r.text, "lxml") results=[] for res in soup.select(".result")[:limit]: a = res.select_one(".result__a") or res.find("a", href=True) href = a.get("href", "") if a else ""; title = _clean_text(a.get_text(" ", strip=True) if a else "") sn = res.select_one(".result__snippet"); snippet = _clean_text(sn.get_text(" ", strip=True) if sn else "") if title or snippet: results.append({"title":title,"url":href,"description":snippet,"content":snippet}) return results except Exception: return [] def web_context(topic: str, limit=6): queries = [topic + " tin tức phân tích bối cảnh", topic] results=[] for q in queries: results = jina_search(q, limit=limit) if results: break if not results: results = duck_context(topic, limit=limit) enriched=[] for item in results[:limit]: content = item.get("content") or item.get("description") or "" if len(content) < 250 and item.get("url", "").startswith("http"): try: page = scrape_any_url(item["url"]) content = (page.get("summary") or "") + " " + page.get("text", "")[:900] except Exception: pass enriched.append({**item, "excerpt": _clean_text(content)[:1200]}) context = "\n".join([f"- {x.get('title') or _domain(x.get('url',''))} ({_domain(x.get('url',''))}): {x.get('excerpt') or x.get('description','')}" for x in enriched if x.get("excerpt") or x.get("title")]) return context, enriched async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 1200): global LAST_QWEN_ERROR, HF_TOKEN HF_TOKEN = _hf_token() if not HF_TOKEN: LAST_QWEN_ERROR = "Không tìm thấy token trong env: HF_TOKEN / HUGGINGFACEHUB_API_TOKEN / HUGGING_FACE_HUB_TOKEN. Hãy Restart Space sau khi thêm secret." return None if not AsyncInferenceClient: LAST_QWEN_ERROR = "Thiếu thư viện huggingface_hub hoặc import lỗi." return None errors=[]; models=[] for m in [QWEN_VL_MODEL, "Qwen/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]: if m and m not in models: models.append(m) for model in models: try: client = AsyncInferenceClient(provider="auto", api_key=HF_TOKEN, timeout=90) content=[] if image_url: content.append({"type":"image_url", "image_url":{"url": image_url}}) content.append({"type":"text", "text": prompt}) messages=[{"role":"system","content":"Bạn là biên tập viên AI tiếng Việt. Nhiệm vụ chính là tóm tắt súc tích nội dung nguồn, không viết lại toàn bài, không lặp ý, không bịa chi tiết."},{"role":"user","content":content}] resp=await client.chat_completion(model=model, messages=messages, max_tokens=max_tokens, temperature=0.45, top_p=0.85) txt=(resp.choices[0].message.content or "").strip() if txt: LAST_QWEN_ERROR=""; return txt except Exception as e: errors.append(f"{model}: {type(e).__name__}: {str(e)[:220]}") LAST_QWEN_ERROR = " | ".join(errors) or "Qwen không trả nội dung." print("[qwen errors]", LAST_QWEN_ERROR) return None def make_post(title, text, image, source_url, kind, sources=None): return {"id":str(int(time.time()*1000))+str(random.randint(100,999)),"title":title,"text":text,"img":image,"url":source_url,"kind":kind,"sources":sources or [],"video":"","ts":int(time.time())} @app.get("/api/ai_wall") def api_ai_wall(): return JSONResponse({"posts":_load_ai_wall()[:80]}) @app.get("/api/ai/status") def api_ai_status(): return JSONResponse({"has_token":bool(_hf_token()),"client_imported":AsyncInferenceClient is not None,"model":QWEN_VL_MODEL,"last_error":LAST_QWEN_ERROR,"tts_ready":gTTS is not None}) @app.post("/api/ai/topic") async def api_ai_topic(request:Request): body=await request.json(); topic=_clean_text(body.get("topic","")) if not topic: return JSONResponse({"error":"missing topic"},status_code=400) ctx,sources=web_context(topic) if not ctx: return JSONResponse({"error":"Không lấy được dữ liệu internet cho chủ đề này. Hãy thử chủ đề cụ thể hơn."},status_code=422) image=pollinations_image_url(topic) prompt=f"""Tóm tắt chủ đề để đăng Tường AI. Chủ đề: {topic} Yêu cầu: - Chỉ tóm tắt các ý chính dựa trên nguồn internet bên dưới. - Không viết lại thành bài dài, không lan man, không lặp lại. - 1 tiêu đề ngắn. - 3-5 gạch đầu dòng, mỗi dòng tối đa 1 câu. - 1 câu kết luận ngắn. - Cuối cùng ghi nguồn tham khảo bằng tên website. Nguồn/bối cảnh internet: {ctx}""" text=await qwen_generate(prompt,image_url=image,max_tokens=900) if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503) post=make_post(topic,text,image,"","topic",sources=sources[:5]); posts=_load_ai_wall(); posts.insert(0,post); _save_ai_wall(posts) return JSONResponse({"post":post}) @app.post("/api/ai/url") async def api_ai_url(request:Request): body=await request.json(); url=_clean_text(body.get("url","")) if not url.startswith("http"): return JSONResponse({"error":"missing url"},status_code=400) try: data=scrape_any_url(url) except Exception as e: return JSONResponse({"error":"Không scrape được URL: "+str(e)[:180]},status_code=422) raw=(data.get("summary","")+"\n"+data.get("text","")).strip() if len(raw)<120: return JSONResponse({"error":"URL không có đủ nội dung để Qwen tóm tắt"},status_code=422) prompt=f"""Tóm tắt bài viết nguồn dưới đây để đăng lên Tường AI VNEWS. Yêu cầu bắt buộc: - Chỉ viết TÓM TẮT theo nội dung chính, không viết lại toàn bộ bài. - Ngắn gọn, xúc tích, cụ thể. - Tránh lặp lại ý và câu chữ trong bài gốc. - Không thêm chi tiết ngoài nguồn. - Độ dài tối đa 6 gạch đầu dòng hoặc 2 đoạn ngắn. - Nếu có nhân vật/sự kiện/thời điểm quan trọng, giữ lại. Tiêu đề gốc: {data.get('title','')} Nguồn scrape: {data.get('via','')} Nội dung gốc: {raw[:16000]}""" text=await qwen_generate(prompt,image_url=data.get("image") or None,max_tokens=900) if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503) post=make_post(data.get("title") or "Bài viết",text,data.get("image") or "",url,"url",sources=[{"title":data.get("title"),"url":url,"excerpt":raw[:500],"via":data.get("via")}]) posts=_load_ai_wall(); posts.insert(0,post); _save_ai_wall(posts) return JSONResponse({"post":post}) @app.post("/api/rewrite_share") async def api_rewrite_share(request:Request): body=await request.json(); url=_clean_text(body.get("url","")) if not url.startswith("http"): return JSONResponse({"error":"missing url"},status_code=400) try: data=scrape_any_url(url) except Exception as e: return JSONResponse({"error":"Không đọc được bài viết: "+str(e)[:180]},status_code=422) raw=(data.get("summary","")+"\n"+data.get("text","")).strip() prompt=f"""Tóm tắt bài viết sau bằng tiếng Việt. Yêu cầu: - Chỉ nêu nội dung chính, không viết lại toàn bài. - Ngắn gọn, cụ thể, dễ hiểu. - Tối đa 5 gạch đầu dòng hoặc 2 đoạn ngắn. - Không lặp ý, không thêm chi tiết ngoài nguồn. Tiêu đề: {data.get('title','')} Nội dung: {raw[:16000]}""" text=await qwen_generate(prompt,image_url=data.get("image") or None,max_tokens=900) if not text: return JSONResponse({"error":"Qwen2.5-VL chưa sẵn sàng: "+LAST_QWEN_ERROR},status_code=503) post=make_post(data.get("title") or "Bài viết",text,data.get("image") or "",url,"rewrite") posts=_load_ai_wall(); posts.insert(0,post); _save_ai_wall(posts) return JSONResponse({"post":post}) def _safe_name(s): return re.sub(r"[^a-zA-Z0-9_-]+", "_", s)[:80] def _wrap_text(text, width=32, max_lines=10): words=_clean_text(text).split(); lines=[]; cur="" for w in words: if len(cur)+len(w)+1<=width: cur=(cur+" "+w).strip() else: if cur: lines.append(cur) cur=w if len(lines)>=max_lines: break if cur and len(lines)target_ratio: new_h=target[1]; new_w=int(new_h*im_ratio) else: new_w=target[0]; new_h=int(new_w/im_ratio) im=im.resize((new_w,new_h)) left=(new_w-target[0])//2; top=(new_h-target[1])//2 im=im.crop((left,top,left+target[0],top+target[1])) bg.paste(im,(0,0)) except Exception: pass draw=ImageDraw.Draw(bg) try: font_title=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",52) font_body=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",40) font_label=ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",32) except Exception: font_title=font_body=font_label=None draw.rectangle((0,780,W,H),fill=(14,14,14)) draw.text((54,830),"VNEWS · Tường AI",fill=(92,184,122),font=font_label) title=_wrap_text(post.get("title",""),24,3) draw.multiline_text((54,900),title,fill=(255,255,255),font=font_title,spacing=10) body=_wrap_text(post.get("text",""),34,10) draw.multiline_text((54,1120),body,fill=(220,220,220),font=font_body,spacing=12) bg.save(out_path,quality=92) def _download_image(url, fallback_topic, out_path): if url: try: r=requests.get(url,headers=HEADERS,timeout=15) if r.status_code==200 and len(r.content)>1000: with open(out_path,"wb") as f:f.write(r.content) return out_path except Exception: pass # fallback generated image gen=pollinations_image_url(fallback_topic) try: r=requests.get(gen,headers=HEADERS,timeout=25) if r.status_code==200 and len(r.content)>1000: with open(out_path,"wb") as f:f.write(r.content) return out_path except Exception: pass if Image: Image.new("RGB",(1080,860),(30,55,42)).save(out_path) return out_path raise RuntimeError("Không tạo được ảnh") def _short_script(post): txt=_clean_text(post.get("text","")) # TTS should read concise summary, not entire huge post. if len(txt)>900: txt=txt[:900].rsplit(" ",1)[0]+"." return f"{post.get('title','')}. {txt}" @app.post("/api/ai/short/{post_id}") def api_ai_short(post_id:str): posts=_load_ai_wall(); post=next((p for p in posts if str(p.get("id"))==str(post_id)),None) if not post: return JSONResponse({"error":"post not found"},status_code=404) os.makedirs(SHORTS_DIR,exist_ok=True) out_mp4=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4") if os.path.exists(out_mp4): post["video"]="/api/ai/short-file/"+post_id _save_ai_wall(posts) return JSONResponse({"video":post["video"]}) if gTTS is None: return JSONResponse({"error":"gTTS chưa sẵn sàng"},status_code=503) work=os.path.join(SHORTS_DIR,_safe_name(post_id)) os.makedirs(work,exist_ok=True) img=os.path.join(work,"image.jpg"); frame=os.path.join(work,"frame.jpg"); audio=os.path.join(work,"voice.mp3") try: _download_image(post.get("img"), post.get("title","AI news"), img) _make_short_frame(post,img,frame) script=_short_script(post) gTTS(script,lang="vi").save(audio) cmd=["ffmpeg","-y","-loop","1","-i",frame,"-i",audio,"-shortest","-c:v","libx264","-tune","stillimage","-pix_fmt","yuv420p","-c:a","aac","-b:a","128k","-vf","scale=1080:1920",out_mp4] subprocess.run(cmd,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=180) post["video"]="/api/ai/short-file/"+post_id _save_ai_wall(posts) return JSONResponse({"video":post["video"]}) except Exception as e: return JSONResponse({"error":"Không tạo được shorts: "+str(e)[:180]},status_code=500) @app.get("/api/ai/short-file/{post_id}") def api_ai_short_file(post_id:str): path=os.path.join(SHORTS_DIR,_safe_name(post_id)+".mp4") if not os.path.exists(path): return JSONResponse({"error":"not found"},status_code=404) return FileResponse(path,media_type="video/mp4",filename=f"vnews-ai-{post_id}.mp4") # Override index route to show Tường AI slide + shorts generation. app.router.routes=[r for r in app.router.routes if not (getattr(r,'path',None)=='/' and 'GET' in getattr(r,'methods',set()))] AI_INJECT=r''' ''' @app.get('/') async def index_ai(): with open('/app/static/index.html','r',encoding='utf-8') as f: html=f.read() return HTMLResponse(html.replace('',AI_INJECT+'\n'))