bep40 commited on
Commit
c139a0b
·
verified ·
1 Parent(s): 8f5df75

Upload ai_ext.py

Browse files
Files changed (1) hide show
  1. ai_ext.py +248 -66
ai_ext.py CHANGED
@@ -3,13 +3,14 @@ Imported by app_v2_entry.py to register /api/rewrite_share, /api/topic_post,
3
  /api/ai_wall, /api/wall, /api/ai/short endpoints on the main FastAPI app.
4
 
5
  Uses main.py's WALL_FILE (wall_posts.json) for unified data store.
 
6
  """
7
- import os, re, json, time, random, html as html_lib, subprocess
8
- from urllib.parse import quote_plus, quote, urlparse
9
- from typing import Optional
10
  import requests
11
  from bs4 import BeautifulSoup
12
- from fastapi import Request
13
  from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
14
 
15
  from main import app
@@ -18,7 +19,6 @@ from main import app
18
  try:
19
  from main import _load_wall, _save_wall, _web_context # noqa: F401
20
  except ImportError:
21
- # Fallback if main doesn't export them
22
  _data_dir = "/data" if os.path.isdir("/data") else "/app/data"
23
  _wall_file = os.path.join(_data_dir, "wall_posts.json")
24
  def _load_wall():
@@ -53,6 +53,10 @@ try:
53
  from PIL import Image, ImageDraw, ImageFont
54
  except Exception:
55
  Image = ImageDraw = ImageFont = None
 
 
 
 
56
 
57
 
58
  def _hf_token():
@@ -64,12 +68,48 @@ def _hf_token():
64
 
65
  HF_TOKEN = _hf_token()
66
  QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
67
- SHORTS_DIR = os.path.join("/data" if os.path.isdir("/data") else "/app/data", "ai_shorts")
68
- HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"vi-VN,vi;q=0.9,en;q=0.8"}
 
 
 
 
69
  LAST_QWEN_ERROR = ""
70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
 
72
- # ===== TEXT HELPERS =====
73
  def _clean_text(s: str) -> str:
74
  s = html_lib.unescape(s or "")
75
  return re.sub(r"\s+", " ", s).strip()
@@ -84,17 +124,13 @@ def _safe_name(s):
84
  return re.sub(r"[^a-zA-Z0-9_-]+", "_", str(s))[:80]
85
 
86
 
87
- # ===== CLEAN AI OUTPUT =====
88
  def _clean_ai_output(text: str) -> str:
89
- """Remove markdown artifacts, instruction leakage, and deduplicate sentences."""
90
  if not text:
91
  return ""
92
- # Remove markdown headings, bold, italic, horizontal rules
93
  text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
94
  text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
95
  text = re.sub(r'\*([^*]+)\*', r'\1', text)
96
  text = re.sub(r'^---+\s*$', '', text, flags=re.MULTILINE)
97
- # Remove common AI instruction leakage phrases
98
  leakage = [
99
  r'Dưới đây là', r'Theo yêu cầu', r'Tôi sẽ', r'Đây là',
100
  r'Nội dung', r'Nhiệm vụ', r'Vai trò', r'Tôi là',
@@ -102,7 +138,6 @@ def _clean_ai_output(text: str) -> str:
102
  ]
103
  for phrase in leakage:
104
  text = re.sub(r'^' + phrase + r'[^\n]*\n', '', text, flags=re.MULTILINE | re.IGNORECASE)
105
- # Deduplicate consecutive identical lines
106
  lines = text.split('\n')
107
  result = []
108
  prev = ""
@@ -113,7 +148,6 @@ def _clean_ai_output(text: str) -> str:
113
  result.append(line)
114
  prev = stripped
115
  text = '\n'.join(result)
116
- # Deduplicate sentences
117
  sentences = re.split(r'(?<=[.!?])\s+', text)
118
  seen_sents = set()
119
  unique_sents = []
@@ -127,6 +161,101 @@ def _clean_ai_output(text: str) -> str:
127
  return ' '.join(unique_sents).strip()
128
 
129
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  # ===== JINA READER =====
131
  def _reader_url(target_url: str) -> str:
132
  safe = quote(target_url, safe=":/?#[]@!$&'()*+,;=%")
@@ -134,7 +263,7 @@ def _reader_url(target_url: str) -> str:
134
 
135
  def jina_reader_markdown(url: str) -> str:
136
  jr = _reader_url(url)
137
- r = requests.get(jr, headers={"Accept":"text/markdown,text/plain,*/*", "X-Return-Format":"markdown", "User-Agent":"Mozilla/5.0"}, timeout=35)
138
  r.raise_for_status()
139
  return r.text or ""
140
 
@@ -165,10 +294,12 @@ def _parse_jina_markdown(md: str, url: str):
165
  break
166
  if not title and paras:
167
  title = paras[0][:90]
168
- return {"url": url, "title": title or url, "summary": paras[0] if paras else "", "text": "\n".join(paras), "image": image, "images": [image] if image else [], "via": "jina"}
 
 
169
 
170
 
171
- # ===== WEB SCRAPE =====
172
  def _best_content_block(soup):
173
  best, best_score = None, 0
174
  for el in soup.find_all(["article", "main", "section", "div"]):
@@ -200,6 +331,13 @@ def scrape_any_url_direct(url: str):
200
  image = img_tag.get("content", "") if img_tag else ""
201
  if image and image.startswith("//"):
202
  image = "https:" + image
 
 
 
 
 
 
 
203
  block = _best_content_block(soup) or soup
204
  paras, seen_p = [], set()
205
  for p in block.find_all("p"):
@@ -211,7 +349,11 @@ def scrape_any_url_direct(url: str):
211
  break
212
  if not title and paras:
213
  title = paras[0][:90]
214
- return {"url": url, "title": title or url, "summary": paras[0] if paras else "", "text": "\n".join(paras), "image": image, "images": [image] if image else [], "via": _domain(url)}
 
 
 
 
215
 
216
  def scrape_any_url(url: str):
217
  """Try direct scrape first, fall back to Jina Reader."""
@@ -228,6 +370,8 @@ def scrape_any_url(url: str):
228
  jr["title"] = data["title"]
229
  if data.get("image"):
230
  jr["image"] = data["image"]
 
 
231
  jr["via"] = data.get("via", _domain(url)) + " + jina"
232
  return jr
233
  except Exception:
@@ -266,7 +410,7 @@ async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens
266
  {"role": "system", "content": (
267
  "Bạn là biên tập viên báo điện tử tiếng Việt. "
268
  "Quy tắc cứng: "
269
- "(1) KHÔNG lặp lại câu/nội dung. "
270
  "(2) KHÔNG dùng Markdown (##, **, ---). "
271
  "(3) KHÔNG viết 'Dưới đây là', 'Tôi sẽ', 'Theo yêu cầu', 'Nhiệm vụ', 'Vai trò'. "
272
  "(4) KHÔNG bịa thông tin ngoài nguồn. "
@@ -287,6 +431,24 @@ async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens
287
  return None
288
 
289
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
  # ===== SHORT VIDEO GENERATION =====
291
  def _download_image(url, fallback_topic, out_path):
292
  if url:
@@ -342,7 +504,6 @@ def _make_short_frame(post, img_path, out_path):
342
  draw.rectangle((0, 780, W, H), fill=(14, 14, 14))
343
  draw.text((54, 830), "VNEWS · Tường AI", fill=(92, 184, 122), font=font_label)
344
  title = post.get("title", "")
345
- # Wrap title
346
  words = title.split(); lines_t = []; cur = ""
347
  for w in words:
348
  if len(cur) + len(w) + 1 <= 24:
@@ -352,7 +513,6 @@ def _make_short_frame(post, img_path, out_path):
352
  cur = w
353
  if cur: lines_t.append(cur)
354
  draw.multiline_text((54, 900), "\n".join(lines_t[:3]), fill=(255, 255, 255), font=font_title, spacing=10)
355
- # Wrap body
356
  body_text = post.get("text", "")
357
  words_b = body_text.split(); lines_b = []; cur_b = ""
358
  for w in words_b:
@@ -373,15 +533,12 @@ def _short_script(post):
373
  txt = txt[:800].rsplit(" ", 1)[0] + "."
374
  return f"{post.get('title', '')}. {txt}"
375
 
376
- def _generate_short_video(post, post_id: str) -> str:
377
- """Generate MP4 short video. Returns video URL or empty string on failure."""
378
  try:
379
  os.makedirs(SHORTS_DIR, exist_ok=True)
380
  out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
381
  if os.path.exists(out_mp4):
382
  return "/api/ai/short-file/" + post_id
383
- if gTTS is None:
384
- return ""
385
  work = os.path.join(SHORTS_DIR, _safe_name(post_id))
386
  os.makedirs(work, exist_ok=True)
387
  img_path = os.path.join(work, "image.jpg")
@@ -390,7 +547,18 @@ def _generate_short_video(post, post_id: str) -> str:
390
  _download_image(post.get("img"), post.get("title", "AI news"), img_path)
391
  _make_short_frame(post, img_path, frame_path)
392
  script = _short_script(post)
393
- gTTS(script, lang="vi").save(audio_path)
 
 
 
 
 
 
 
 
 
 
 
394
  cmd = ["ffmpeg", "-y", "-loop", "1", "-i", frame_path, "-i", audio_path,
395
  "-shortest", "-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
396
  "-c:a", "aac", "-b:a", "128k", "-vf", "scale=1080:1920", out_mp4]
@@ -402,25 +570,30 @@ def _generate_short_video(post, post_id: str) -> str:
402
 
403
 
404
  # ===== MAKE POST =====
405
- def make_post(title, text, image, source_url, kind, sources=None):
406
  return {
407
  "id": str(int(time.time() * 1000)) + str(random.randint(100, 999)),
408
  "title": title, "text": text, "img": image, "url": source_url,
409
- "kind": kind, "sources": sources or [], "video": "", "ts": int(time.time())
 
410
  }
411
 
412
 
413
  # ===== SHARED PROMPT BUILDER =====
414
- def _build_rewrite_prompt(title: str, raw: str) -> str:
 
 
 
415
  return f"""Tóm tắt bài viết sau thành bài ngắn gọn đăng Tường AI (tối đa 200 từ).
416
 
417
- YÊU CẦU:
418
  - Chỉ nêu các Ý CHÍNH, không viết chi tiết thừa.
419
- - KHÔNG lặp lại câu/nội dung.
420
  - KHÔNG dùng Markdown (##, **, ---).
421
  - KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu".
422
- - Viết thành đoạn văn thuần, mạch lạc.
423
- - Giữ sự thật, không bịa.
 
424
 
425
  Tiêu đề gốc: {title}
426
 
@@ -431,9 +604,9 @@ Nội dung gốc:
431
  def _build_topic_prompt(topic: str, ctx: str) -> str:
432
  return f"""Viết bài tổng hợp NGẮN GỌN về chủ đề: "{topic}" (tối đa 200 từ).
433
 
434
- YÊU CẦU:
435
  - Chỉ tổng hợp các Ý CHÍNH từ nguồn.
436
- - KHÔNG lặp lại nội dung.
437
  - KHÔNG sao chép nguyên văn; diễn đạt lại.
438
  - KHÔNG dùng Markdown (##, **, ---).
439
  - KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu".
@@ -458,19 +631,18 @@ async def api_rewrite_share(request: Request):
458
  raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
459
  if len(raw) < 60:
460
  return JSONResponse({"error": "Bài viết quá ngắn để tóm tắt"}, status_code=422)
461
-
462
- prompt = _build_rewrite_prompt(data.get("title", ""), raw)
463
- text = await qwen_generate(prompt, image_url=data.get("image") or None, max_tokens=700)
464
  if not text:
465
  return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
466
  text = _clean_ai_output(text)
467
- post = make_post(data.get("title") or "Bài viết", text, data.get("image") or "", url, "rewrite")
468
-
469
- # Auto-generate short video in background
470
- video_url = _generate_short_video(post, post["id"])
471
  if video_url:
472
  post["video"] = video_url
473
-
474
  posts = _load_wall()
475
  posts.insert(0, post)
476
  _save_wall(posts)
@@ -490,18 +662,18 @@ async def api_url_wall(request: Request):
490
  raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
491
  if len(raw) < 60:
492
  return JSONResponse({"error": "URL không có đủ nội dung"}, status_code=422)
493
-
494
- prompt = _build_rewrite_prompt(data.get("title", ""), raw)
495
- text = await qwen_generate(prompt, image_url=data.get("image") or None, max_tokens=700)
496
  if not text:
497
  return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
498
  text = _clean_ai_output(text)
499
- post = make_post(data.get("title") or "Bài viết", text, data.get("image") or "", url, "url")
500
-
501
- video_url = _generate_short_video(post, post["id"])
 
502
  if video_url:
503
  post["video"] = video_url
504
-
505
  posts = _load_wall()
506
  posts.insert(0, post)
507
  _save_wall(posts)
@@ -514,11 +686,9 @@ async def api_topic_post(request: Request):
514
  topic = _clean_text(body.get("topic", ""))
515
  if not topic:
516
  return JSONResponse({"error": "missing topic"}, status_code=400)
517
-
518
  ctx = _web_context(topic)
519
  if not ctx:
520
  return JSONResponse({"error": "Không lấy được dữ liệu cho chủ đề này"}, status_code=422)
521
-
522
  image = pollinations_image_url(topic)
523
  prompt = _build_topic_prompt(topic, ctx)
524
  text = await qwen_generate(prompt, image_url=image, max_tokens=700)
@@ -526,11 +696,9 @@ async def api_topic_post(request: Request):
526
  return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
527
  text = _clean_ai_output(text)
528
  post = make_post(topic, text, image, "", "topic")
529
-
530
- video_url = _generate_short_video(post, post["id"])
531
  if video_url:
532
  post["video"] = video_url
533
-
534
  posts = _load_wall()
535
  posts.insert(0, post)
536
  _save_wall(posts)
@@ -540,37 +708,38 @@ async def api_topic_post(request: Request):
540
  # ===== WALL ENDPOINTS =====
541
  @app.get("/api/ai_wall")
542
  def api_ai_wall():
543
- """Legacy endpoint — same data as /api/wall."""
544
  return JSONResponse({"posts": _load_wall()[:80]})
545
 
546
-
547
  @app.get("/api/wall")
548
  def api_wall():
549
- """Unified wall endpoint — same data store as main.py's wall_posts.json."""
550
  return JSONResponse({"posts": _load_wall()[:80]})
551
 
552
 
 
553
  @app.post("/api/ai/short/{post_id}")
554
- def api_ai_short(post_id: str):
555
- """Generate (or retrieve cached) short video for a wall post."""
 
 
 
 
556
  posts = _load_wall()
557
  post = next((p for p in posts if str(p.get("id")) == str(post_id)), None)
558
  if not post:
559
  return JSONResponse({"error": "post not found"}, status_code=404)
560
  os.makedirs(SHORTS_DIR, exist_ok=True)
561
  out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
562
- if os.path.exists(out_mp4):
563
  video_url = "/api/ai/short-file/" + post_id
564
- # Update post in store
565
  for i, p in enumerate(posts):
566
  if str(p.get("id")) == str(post_id):
567
  posts[i]["video"] = video_url
568
  break
569
  _save_wall(posts)
570
  return JSONResponse({"video": video_url})
571
- if gTTS is None:
572
- return JSONResponse({"error": "gTTS chưa sẵn sàng"}, status_code=503)
573
- video_url = _generate_short_video(post, post_id)
574
  if video_url:
575
  for i, p in enumerate(posts):
576
  if str(p.get("id")) == str(post_id):
@@ -596,5 +765,18 @@ def api_ai_status():
596
  "client_imported": AsyncInferenceClient is not None,
597
  "model": QWEN_VL_MODEL,
598
  "last_error": LAST_QWEN_ERROR,
599
- "tts_ready": gTTS is not None
 
 
 
 
 
 
 
 
 
 
 
 
 
600
  })
 
3
  /api/ai_wall, /api/wall, /api/ai/short endpoints on the main FastAPI app.
4
 
5
  Uses main.py's WALL_FILE (wall_posts.json) for unified data store.
6
+ TTS: edge-tts (HoaiMy female, NamMinh male) with speed control + gTTS fallback.
7
  """
8
+ import os, re, json, time, random, html as html_lib, subprocess, asyncio
9
+ from urllib.parse import quote_plus, quote, urlparse, urljoin
10
+ from typing import Optional, List, Dict
11
  import requests
12
  from bs4 import BeautifulSoup
13
+ from fastapi import Request, Query
14
  from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
15
 
16
  from main import app
 
19
  try:
20
  from main import _load_wall, _save_wall, _web_context # noqa: F401
21
  except ImportError:
 
22
  _data_dir = "/data" if os.path.isdir("/data") else "/app/data"
23
  _wall_file = os.path.join(_data_dir, "wall_posts.json")
24
  def _load_wall():
 
53
  from PIL import Image, ImageDraw, ImageFont
54
  except Exception:
55
  Image = ImageDraw = ImageFont = None
56
+ try:
57
+ import edge_tts
58
+ except Exception:
59
+ edge_tts = None
60
 
61
 
62
  def _hf_token():
 
68
 
69
  HF_TOKEN = _hf_token()
70
  QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
71
+ DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
72
+ SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts")
73
+ HEADERS = {
74
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
75
+ "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"
76
+ }
77
  LAST_QWEN_ERROR = ""
78
 
79
+ # ===== TTS VOICE CONFIG =====
80
+ TTS_VOICES = {
81
+ "hoaimy": {"id": "vi-VN-HoaiMyNeural", "gender": "female", "name": "Hoài My (Nữ)"},
82
+ "namminh": {"id": "vi-VN-NamMinhNeural", "gender": "male", "name": "Nam Minh (Nam)"},
83
+ }
84
+ TTS_DEFAULT_VOICE = "hoaimy"
85
+ TTS_DEFAULT_SPEED = 1.2
86
+
87
+ TOPIC_VOICE_MAP = {
88
+ "bóng đá": "namminh", "thể thao": "namminh", "world cup": "namminh",
89
+ "premier league": "namminh", "champions league": "namminh", "la liga": "namminh",
90
+ "serie a": "namminh", "bundesliga": "namminh", "v-league": "namminh",
91
+ "tennis": "namminh", "olympic": "namminh", "f1": "namminh", "moto": "namminh",
92
+ "sức khỏe": "hoaimy", "làm đẹp": "hoaimy", "giải trí": "hoaimy",
93
+ "âm nhạc": "hoaimy", "phim": "hoaimy", "thời trang": "hoaimy",
94
+ "ẩm thực": "hoaimy", "du lịch": "hoaimy", "gia đình": "hoaimy",
95
+ "tình yêu": "hoaimy", "hôn nhân": "hoaimy", "mẹ và bé": "hoaimy",
96
+ "công nghệ": "namminh", "ai": "namminh", "robot": "namminh",
97
+ "khoa học": "namminh", "vũ trụ": "namminh", "điện thoại": "namminh",
98
+ "laptop": "namminh", "game": "namminh",
99
+ "chính trị": "namminh", "kinh tế": "namminh", "tài chính": "namminh",
100
+ "chứng khoán": "namminh", "ngân hàng": "namminh", "thị trường": "namminh",
101
+ "xã hội": "namminh", "pháp luật": "namminh", "giáo dục": "namminh",
102
+ }
103
+
104
+
105
+ def _detect_voice_for_topic(title: str, text: str) -> str:
106
+ combined = (title + " " + text[:500]).lower()
107
+ for keyword, voice_id in TOPIC_VOICE_MAP.items():
108
+ if keyword in combined:
109
+ return voice_id
110
+ return TTS_DEFAULT_VOICE
111
+
112
 
 
113
  def _clean_text(s: str) -> str:
114
  s = html_lib.unescape(s or "")
115
  return re.sub(r"\s+", " ", s).strip()
 
124
  return re.sub(r"[^a-zA-Z0-9_-]+", "_", str(s))[:80]
125
 
126
 
 
127
  def _clean_ai_output(text: str) -> str:
 
128
  if not text:
129
  return ""
 
130
  text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
131
  text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
132
  text = re.sub(r'\*([^*]+)\*', r'\1', text)
133
  text = re.sub(r'^---+\s*$', '', text, flags=re.MULTILINE)
 
134
  leakage = [
135
  r'Dưới đây là', r'Theo yêu cầu', r'Tôi sẽ', r'Đây là',
136
  r'Nội dung', r'Nhiệm vụ', r'Vai trò', r'Tôi là',
 
138
  ]
139
  for phrase in leakage:
140
  text = re.sub(r'^' + phrase + r'[^\n]*\n', '', text, flags=re.MULTILINE | re.IGNORECASE)
 
141
  lines = text.split('\n')
142
  result = []
143
  prev = ""
 
148
  result.append(line)
149
  prev = stripped
150
  text = '\n'.join(result)
 
151
  sentences = re.split(r'(?<=[.!?])\s+', text)
152
  seen_sents = set()
153
  unique_sents = []
 
161
  return ' '.join(unique_sents).strip()
162
 
163
 
164
+ # ===== EXTRACT ALL IMAGES FROM ARTICLE =====
165
+ def _extract_all_images(soup, base_url: str) -> List[Dict]:
166
+ images = []
167
+ seen_urls = set()
168
+ skip_patterns = [
169
+ "avatar", "icon", "logo", "button", "banner-ad", "tracking",
170
+ "beacon", "pixel", "1x1", "spacer", "emoji", "sprite", "placeholder",
171
+ "advertisement", "ads", "widget", "sidebar", "footer-logo",
172
+ ]
173
+
174
+ def _add_image(src: str, alt: str = "", source_tag: str = "img"):
175
+ if not src or src.startswith("data:"):
176
+ return
177
+ abs_url = urljoin(base_url, src.strip())
178
+ if abs_url in seen_urls:
179
+ return
180
+ if any(p in abs_url.lower() for p in skip_patterns):
181
+ return
182
+ try:
183
+ parsed = urlparse(abs_url)
184
+ path = parsed.path.lower()
185
+ if any(path.endswith(ext) for ext in ['.svg', '.ico', '.gif']):
186
+ return
187
+ except Exception:
188
+ pass
189
+ seen_urls.add(abs_url)
190
+ images.append({"url": abs_url, "alt": alt, "source": source_tag})
191
+
192
+ # Strategy 1: Standard <img> tags with all lazy-load attributes
193
+ for img in soup.find_all("img"):
194
+ src = (img.get("src") or img.get("data-src") or img.get("data-lazy-src") or
195
+ img.get("data-original") or img.get("data-srcset", "").split(",")[0].strip().split(" ")[0])
196
+ _add_image(src, alt=img.get("alt", ""), source_tag="img")
197
+
198
+ # Strategy 2: srcset on <img>
199
+ for img in soup.find_all("img", srcset=True):
200
+ for part in img["srcset"].split(","):
201
+ part = part.strip()
202
+ if part:
203
+ _add_image(part.split(" ")[0], alt=img.get("alt", ""), source_tag="srcset")
204
+
205
+ # Strategy 3: <picture> with <source>
206
+ for picture in soup.find_all("picture"):
207
+ for source in picture.find_all("source"):
208
+ srcset = source.get("srcset", "")
209
+ for part in srcset.split(","):
210
+ part = part.strip()
211
+ if part:
212
+ _add_image(part.split(" ")[0], source_tag="picture/srcset")
213
+ fallback_img = picture.find("img")
214
+ if fallback_img:
215
+ _add_image(
216
+ fallback_img.get("src") or fallback_img.get("data-src"),
217
+ alt=fallback_img.get("alt", ""),
218
+ source_tag="picture/img"
219
+ )
220
+
221
+ # Strategy 4: WordPress CMS patterns
222
+ for img in soup.find_all("img", class_=re.compile(r"wp-image|size-large|size-full|aligncenter")):
223
+ _add_image(img.get("data-src") or img.get("src"),
224
+ alt=img.get("alt", ""), source_tag="wp-image")
225
+
226
+ # Strategy 5: Background images in style attributes
227
+ for tag in soup.find_all(style=re.compile(r"background-image")):
228
+ for m in re.findall(r'url\(["\']?(.*?)["\']?\)', tag.get("style", "")):
229
+ _add_image(m, source_tag="background-style")
230
+
231
+ # Strategy 6: og:image (featured/hero image)
232
+ og_image = soup.find("meta", property="og:image")
233
+ if og_image and og_image.get("content"):
234
+ _add_image(og_image["content"], source_tag="og:image")
235
+
236
+ # Strategy 7: twitter:image
237
+ tw_image = soup.find("meta", attrs={"name": "twitter:image"})
238
+ if tw_image and tw_image.get("content"):
239
+ _add_image(tw_image["content"], source_tag="twitter:image")
240
+
241
+ # Strategy 8: <figure> with <figcaption>
242
+ for figure in soup.find_all("figure"):
243
+ img = figure.find("img")
244
+ if img:
245
+ src = img.get("data-src") or img.get("src")
246
+ figcaption = figure.find("figcaption")
247
+ alt = figcaption.get_text(strip=True) if figcaption else img.get("alt", "")
248
+ _add_image(src, alt=alt, source_tag="figure")
249
+
250
+ # Strategy 9: <a> tags linking to images
251
+ for a in soup.find_all("a", href=True):
252
+ href = a["href"]
253
+ if any(href.lower().endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".webp", ".gif"]):
254
+ _add_image(href, alt=a.get_text(strip=True)[:80], source_tag="link")
255
+
256
+ return images
257
+
258
+
259
  # ===== JINA READER =====
260
  def _reader_url(target_url: str) -> str:
261
  safe = quote(target_url, safe=":/?#[]@!$&'()*+,;=%")
 
263
 
264
  def jina_reader_markdown(url: str) -> str:
265
  jr = _reader_url(url)
266
+ r = requests.get(jr, headers={"Accept": "text/markdown,text/plain,*/*", "X-Return-Format": "markdown", "User-Agent": "Mozilla/5.0"}, timeout=35)
267
  r.raise_for_status()
268
  return r.text or ""
269
 
 
294
  break
295
  if not title and paras:
296
  title = paras[0][:90]
297
+ return {"url": url, "title": title or url, "summary": paras[0] if paras else "",
298
+ "text": "\n".join(paras), "image": image,
299
+ "images": [image] if image else [], "via": "jina"}
300
 
301
 
302
+ # ===== WEB SCRAPE (with full image extraction) =====
303
  def _best_content_block(soup):
304
  best, best_score = None, 0
305
  for el in soup.find_all(["article", "main", "section", "div"]):
 
331
  image = img_tag.get("content", "") if img_tag else ""
332
  if image and image.startswith("//"):
333
  image = "https:" + image
334
+ all_images = _extract_all_images(soup, url)
335
+ image_urls = [img["url"] for img in all_images]
336
+ if image and image not in image_urls:
337
+ image_urls.insert(0, image)
338
+ elif image in image_urls:
339
+ image_urls.remove(image)
340
+ image_urls.insert(0, image)
341
  block = _best_content_block(soup) or soup
342
  paras, seen_p = [], set()
343
  for p in block.find_all("p"):
 
349
  break
350
  if not title and paras:
351
  title = paras[0][:90]
352
+ return {
353
+ "url": url, "title": title or url, "summary": paras[0] if paras else "",
354
+ "text": "\n".join(paras), "image": image_urls[0] if image_urls else "",
355
+ "images": image_urls, "via": _domain(url)
356
+ }
357
 
358
  def scrape_any_url(url: str):
359
  """Try direct scrape first, fall back to Jina Reader."""
 
370
  jr["title"] = data["title"]
371
  if data.get("image"):
372
  jr["image"] = data["image"]
373
+ if data.get("images"):
374
+ jr["images"] = data["images"]
375
  jr["via"] = data.get("via", _domain(url)) + " + jina"
376
  return jr
377
  except Exception:
 
410
  {"role": "system", "content": (
411
  "Bạn là biên tập viên báo điện tử tiếng Việt. "
412
  "Quy tắc cứng: "
413
+ "(1) KHÔNG lặp lại câu/nội dung đã viết. Mỗi ý chỉ xuất hiện 1 lần. "
414
  "(2) KHÔNG dùng Markdown (##, **, ---). "
415
  "(3) KHÔNG viết 'Dưới đây là', 'Tôi sẽ', 'Theo yêu cầu', 'Nhiệm vụ', 'Vai trò'. "
416
  "(4) KHÔNG bịa thông tin ngoài nguồn. "
 
431
  return None
432
 
433
 
434
+ # ===== TTS GENERATION =====
435
+ async def _generate_tts_edge(text: str, voice_id: str, speed: float, out_path: str):
436
+ """Generate TTS using edge-tts with voice and speed control."""
437
+ if edge_tts is None:
438
+ raise RuntimeError("edge-tts chưa cài đặt")
439
+ voice = TTS_VOICES.get(voice_id, TTS_VOICES[TTS_DEFAULT_VOICE])["id"]
440
+ pct = int((speed - 1.0) * 100)
441
+ rate = f"+{pct}%" if pct >= 0 else f"{pct}%"
442
+ communicate = edge_tts.Communicate(text, voice, rate=rate)
443
+ await communicate.save(out_path)
444
+
445
+ def _generate_tts_gtts(text: str, out_path: str):
446
+ """Fallback TTS using gTTS (no voice/speed control)."""
447
+ if gTTS is None:
448
+ raise RuntimeError("gTTS chưa cài đặt")
449
+ gTTS(text, lang="vi").save(out_path)
450
+
451
+
452
  # ===== SHORT VIDEO GENERATION =====
453
  def _download_image(url, fallback_topic, out_path):
454
  if url:
 
504
  draw.rectangle((0, 780, W, H), fill=(14, 14, 14))
505
  draw.text((54, 830), "VNEWS · Tường AI", fill=(92, 184, 122), font=font_label)
506
  title = post.get("title", "")
 
507
  words = title.split(); lines_t = []; cur = ""
508
  for w in words:
509
  if len(cur) + len(w) + 1 <= 24:
 
513
  cur = w
514
  if cur: lines_t.append(cur)
515
  draw.multiline_text((54, 900), "\n".join(lines_t[:3]), fill=(255, 255, 255), font=font_title, spacing=10)
 
516
  body_text = post.get("text", "")
517
  words_b = body_text.split(); lines_b = []; cur_b = ""
518
  for w in words_b:
 
533
  txt = txt[:800].rsplit(" ", 1)[0] + "."
534
  return f"{post.get('title', '')}. {txt}"
535
 
536
+ async def _generate_short_video(post, post_id: str, voice_id: str = None, speed: float = None) -> str:
 
537
  try:
538
  os.makedirs(SHORTS_DIR, exist_ok=True)
539
  out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
540
  if os.path.exists(out_mp4):
541
  return "/api/ai/short-file/" + post_id
 
 
542
  work = os.path.join(SHORTS_DIR, _safe_name(post_id))
543
  os.makedirs(work, exist_ok=True)
544
  img_path = os.path.join(work, "image.jpg")
 
547
  _download_image(post.get("img"), post.get("title", "AI news"), img_path)
548
  _make_short_frame(post, img_path, frame_path)
549
  script = _short_script(post)
550
+ if voice_id is None:
551
+ voice_id = _detect_voice_for_topic(post.get("title", ""), post.get("text", ""))
552
+ if speed is None:
553
+ speed = TTS_DEFAULT_SPEED
554
+ try:
555
+ await _generate_tts_edge(script, voice_id, speed, audio_path)
556
+ except Exception as e:
557
+ print(f"[TTS edge-tts error] {e}, falling back to gTTS")
558
+ if gTTS:
559
+ _generate_tts_gtts(script, audio_path)
560
+ else:
561
+ return ""
562
  cmd = ["ffmpeg", "-y", "-loop", "1", "-i", frame_path, "-i", audio_path,
563
  "-shortest", "-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
564
  "-c:a", "aac", "-b:a", "128k", "-vf", "scale=1080:1920", out_mp4]
 
570
 
571
 
572
  # ===== MAKE POST =====
573
+ def make_post(title, text, image, source_url, kind, sources=None, images=None):
574
  return {
575
  "id": str(int(time.time() * 1000)) + str(random.randint(100, 999)),
576
  "title": title, "text": text, "img": image, "url": source_url,
577
+ "kind": kind, "sources": sources or [], "video": "",
578
+ "images": images or [], "ts": int(time.time())
579
  }
580
 
581
 
582
  # ===== SHARED PROMPT BUILDER =====
583
+ def _build_rewrite_prompt(title: str, raw: str, images: List[str] = None) -> str:
584
+ image_info = ""
585
+ if images and len(images) > 1:
586
+ image_info = f"\n\nBài viết có {len(images)} ảnh. Hãy tham khảo nội dung ảnh để bổ sung thông tin cho bài viết (nếu phù hợp)."
587
  return f"""Tóm tắt bài viết sau thành bài ngắn gọn đăng Tường AI (tối đa 200 từ).
588
 
589
+ YÊU CẦU QUAN TRỌNG:
590
  - Chỉ nêu các Ý CHÍNH, không viết chi tiết thừa.
591
+ - KHÔNG lặp lại câu/nội dung. Mỗi ý chỉ xuất hiện 1 lần.
592
  - KHÔNG dùng Markdown (##, **, ---).
593
  - KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu".
594
+ - Viết thành đoạn văn thuần, mạch lạc, dễ đọc.
595
+ - Giữ sự thật, không bịa thông tin.
596
+ - Tóm tắt súc tích, đúng trọng tâm.{image_info}
597
 
598
  Tiêu đề gốc: {title}
599
 
 
604
  def _build_topic_prompt(topic: str, ctx: str) -> str:
605
  return f"""Viết bài tổng hợp NGẮN GỌN về chủ đề: "{topic}" (tối đa 200 từ).
606
 
607
+ YÊU CẦU QUAN TRỌNG:
608
  - Chỉ tổng hợp các Ý CHÍNH từ nguồn.
609
+ - KHÔNG lặp lại nội dung. Mỗi ý chỉ xuất hiện 1 lần.
610
  - KHÔNG sao chép nguyên văn; diễn đạt lại.
611
  - KHÔNG dùng Markdown (##, **, ---).
612
  - KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu".
 
631
  raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
632
  if len(raw) < 60:
633
  return JSONResponse({"error": "Bài viết quá ngắn để tóm tắt"}, status_code=422)
634
+ images = data.get("images", [])
635
+ prompt = _build_rewrite_prompt(data.get("title", ""), raw, images)
636
+ text = await qwen_generate(prompt, image_url=images[0] if images else None, max_tokens=700)
637
  if not text:
638
  return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
639
  text = _clean_ai_output(text)
640
+ post = make_post(data.get("title") or "Bài viết", text,
641
+ images[0] if images else data.get("image", ""),
642
+ url, "rewrite", images=images)
643
+ video_url = await _generate_short_video(post, post["id"])
644
  if video_url:
645
  post["video"] = video_url
 
646
  posts = _load_wall()
647
  posts.insert(0, post)
648
  _save_wall(posts)
 
662
  raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
663
  if len(raw) < 60:
664
  return JSONResponse({"error": "URL không có đủ nội dung"}, status_code=422)
665
+ images = data.get("images", [])
666
+ prompt = _build_rewrite_prompt(data.get("title", ""), raw, images)
667
+ text = await qwen_generate(prompt, image_url=images[0] if images else None, max_tokens=700)
668
  if not text:
669
  return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
670
  text = _clean_ai_output(text)
671
+ post = make_post(data.get("title") or "Bài viết", text,
672
+ images[0] if images else data.get("image", ""),
673
+ url, "url", images=images)
674
+ video_url = await _generate_short_video(post, post["id"])
675
  if video_url:
676
  post["video"] = video_url
 
677
  posts = _load_wall()
678
  posts.insert(0, post)
679
  _save_wall(posts)
 
686
  topic = _clean_text(body.get("topic", ""))
687
  if not topic:
688
  return JSONResponse({"error": "missing topic"}, status_code=400)
 
689
  ctx = _web_context(topic)
690
  if not ctx:
691
  return JSONResponse({"error": "Không lấy được dữ liệu cho chủ đề này"}, status_code=422)
 
692
  image = pollinations_image_url(topic)
693
  prompt = _build_topic_prompt(topic, ctx)
694
  text = await qwen_generate(prompt, image_url=image, max_tokens=700)
 
696
  return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
697
  text = _clean_ai_output(text)
698
  post = make_post(topic, text, image, "", "topic")
699
+ video_url = await _generate_short_video(post, post["id"])
 
700
  if video_url:
701
  post["video"] = video_url
 
702
  posts = _load_wall()
703
  posts.insert(0, post)
704
  _save_wall(posts)
 
708
  # ===== WALL ENDPOINTS =====
709
  @app.get("/api/ai_wall")
710
  def api_ai_wall():
 
711
  return JSONResponse({"posts": _load_wall()[:80]})
712
 
 
713
  @app.get("/api/wall")
714
  def api_wall():
 
715
  return JSONResponse({"posts": _load_wall()[:80]})
716
 
717
 
718
+ # ===== SHORT VIDEO ENDPOINT (with voice + speed params) =====
719
  @app.post("/api/ai/short/{post_id}")
720
+ async def api_ai_short(post_id: str, voice: str = Query(default=None), speed: float = Query(default=None)):
721
+ """Generate (or retrieve cached) short video for a wall post.
722
+ Query params:
723
+ - voice: 'hoaimy' (female) | 'namminh' (male) | auto-detect if not specified
724
+ - speed: float (default 1.2), e.g. 1.0=normal, 1.2=fast, 0.8=slow
725
+ """
726
  posts = _load_wall()
727
  post = next((p for p in posts if str(p.get("id")) == str(post_id)), None)
728
  if not post:
729
  return JSONResponse({"error": "post not found"}, status_code=404)
730
  os.makedirs(SHORTS_DIR, exist_ok=True)
731
  out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
732
+ if os.path.exists(out_mp4) and voice is None and speed is None:
733
  video_url = "/api/ai/short-file/" + post_id
 
734
  for i, p in enumerate(posts):
735
  if str(p.get("id")) == str(post_id):
736
  posts[i]["video"] = video_url
737
  break
738
  _save_wall(posts)
739
  return JSONResponse({"video": video_url})
740
+ if voice is not None and voice not in TTS_VOICES:
741
+ return JSONResponse({"error": f"voice không hợp lệ. Chọn: {list(TTS_VOICES.keys())}"}, status_code=400)
742
+ video_url = await _generate_short_video(post, post_id, voice_id=voice, speed=speed)
743
  if video_url:
744
  for i, p in enumerate(posts):
745
  if str(p.get("id")) == str(post_id):
 
765
  "client_imported": AsyncInferenceClient is not None,
766
  "model": QWEN_VL_MODEL,
767
  "last_error": LAST_QWEN_ERROR,
768
+ "tts_ready": gTTS is not None or edge_tts is not None,
769
+ "tts_engine": "edge-tts" if edge_tts else ("gtts" if gTTS else "none"),
770
+ "tts_voices": {k: v["name"] for k, v in TTS_VOICES.items()},
771
+ "tts_default_speed": TTS_DEFAULT_SPEED,
772
+ })
773
+
774
+
775
+ @app.get("/api/ai/voices")
776
+ def api_ai_voices():
777
+ return JSONResponse({
778
+ "voices": {k: {"name": v["name"], "gender": v["gender"]} for k, v in TTS_VOICES.items()},
779
+ "default_voice": TTS_DEFAULT_VOICE,
780
+ "default_speed": TTS_DEFAULT_SPEED,
781
+ "topic_voice_map": TOPIC_VOICE_MAP,
782
  })