diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 40f60e1de89e0f904c11d302058de8c039f587a9..0000000000000000000000000000000000000000
--- a/.gitignore
+++ /dev/null
@@ -1,6 +0,0 @@
-__pycache__/
-*.pyc
-data/
-.data
-.huggingface/
-.restart_trigger
diff --git a/CHANGELOG.md b/CHANGELOG.md
deleted file mode 100644
index bed2a391f57aae4c106228b98824a5a07c71913b..0000000000000000000000000000000000000000
--- a/CHANGELOG.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# FPT Play Stream Selector Update
-
-Added FPT Play channel with stream selector UI similar to VTV6:
-
-- New tab "FPT" (orange themed) in the channel tabs
-- Stream selector with 4 sources:
- 1. 🌐 Web FPT Play (iframe)
- 2. 📡 HLS Proxy (via /api/proxy/m3u8)
- 3. 🔗 HLS Direct
- 4. 📺 HD1.xemtv.net (iframe from LINK 1)
-- Backend vtv_api.py now returns stream_selectors for fpt-the-thao channel
-- Frontend handles stream switching automatically when FPT tab is active
-
-## Changes
-- `static/vtv_init.js`: Added FPT tab + stream selector UI logic
-- `vtv_api.py`: Added FPT Play endpoint responses with stream_selectors data
diff --git a/Dockerfile b/Dockerfile
index fd799e0b9eca048c628530076b0a14544944a46b..bbedafd73d632f813392e62974ca6a580ab4d7bf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,48 +1,15 @@
-FROM python:3.12-slim
+FROM python:3.11-slim
WORKDIR /app
-RUN echo "[BUILD] step1: apt-get update+install ffmpeg + Vietnamese fonts" && \
- apt-get update && apt-get install -y --no-install-recommends \
- ffmpeg \
- fonts-dejavu-core \
- fonts-noto \
- fonts-noto-cjk \
- fonts-noto-color-emoji \
- fonts-liberation \
- fonts-freefont-ttf \
- libfreetype6 \
- && rm -rf /var/lib/apt/lists/* && \
- echo "[BUILD] step1 done"
-
-RUN echo "[BUILD] step2: pip base pkgs (bs4/lxml)" && \
- pip install --no-cache-dir "beautifulsoup4>=4.12" lxml && \
- echo "[BUILD] step2 done"
-
-RUN echo "[BUILD] step3: pip main pkgs" && \
- pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4 jinja2 yt-dlp huggingface_hub gTTS pillow edge-tts python-dateutil httpx pycryptodome && \
- echo "[BUILD] step3 done"
-
-COPY requirements.txt .
-RUN echo "[BUILD] step4: pip requirements.txt" && \
- pip install --no-cache-dir -r requirements.txt || true && \
- echo "[BUILD] step4 done"
+RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg fonts-dejavu-core && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4 lxml jinja2 yt-dlp huggingface_hub gTTS pillow edge-tts
COPY . .
-EXPOSE 7860
-RUN echo "[BUILD] step5: setup Vietnamese font symlink" && \
- mkdir -p /usr/share/fonts/truetype/vn && \
- # Prefer Noto Sans for Vietnamese - it has full diacritic support
- if [ -f /usr/share/fonts/truetype/noto/NotoSans-Regular.ttf ]; then \
- ln -sf /usr/share/fonts/truetype/noto/NotoSans-Regular.ttf /usr/share/fonts/truetype/vn/VNFont.ttf; \
- elif [ -f /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf ]; then \
- ln -sf /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf /usr/share/fonts/truetype/vn/VNFont.ttf; \
- ln -sf /usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf /usr/share/fonts/truetype/vn/VNFont-Bold.ttf; \
- fi; \
- fc-cache -f -v || true; \
- date > /app/.build_done && \
- echo "[BUILD] step5 done"
+EXPOSE 7860
-CMD ["uvicorn", "_run:app", "--host", "0.0.0.0", "--port", "7860"]
-# v3.0-vn-font-fix-short-video-2026-07-19
+# CRITICAL: Use app_v2_entry.py which imports ONLY main.py (core APIs)
+# This bypasses the entire injection chain (app_run → app_final → ai_runtime_final6)
+# Guarantees index_v2.html is served clean without old JS patches
+CMD ["uvicorn", "app_v2_entry:app", "--host", "0.0.0.0", "--port", "7860"]
diff --git a/README.md b/README.md
index f65ec063b0184f66ec5c41d94307c4f38f4c0a38..ebcd71c672947f56bc568d773f5762789310a420 100644
--- a/README.md
+++ b/README.md
@@ -9,50 +9,5 @@ tags:
- ml-intern
---
-# VNEWS - Tin Tức Việt Nam
-
-**v18 - FIXED VTV2/VTV3/VTV6/VTV9 stream hanging**
-
-## 🔧 Changes in v18 (2026-07-06)
-- **VTV2, VTV3, VTV6, VTV9**: Skip expired ssaimh CDN token → immediately fall through to sv2.xemtivitop.com
-- **15+ extraction patterns** for m3u8 URL (up from 5), including: file:, src=, source:, player.src(), hls.loadSource(), href=, ``, url:, window.location, iframe follow (3 levels deep), base64 decode
-- **Backup CDN** `tv.mediacdn.vn` for VTV2/VTV3/VTV6/VTV9
-- **Fast timeout** 5s for CDN, 12s for PHP endpoints (was 15s each = 60s+ total)
-- **sv2.xemtivitop.com** re-prioritized to check BEFORE xemtv.us
-- **Iframe chain following**: if a PHP page returns an iframe → follow it up to 3 levels to find the m3u8
-
-## Features:
-- 📰 News from VnExpress (10 categories) + GenK AI
-- ⚽ Livescore from bongda.com.vn (live, today, upcoming, results, standings)
-- 🎬 Football highlights from xemlaibongda.top (8 leagues)
-- 📺 VTV live channels (VTV1→VTV10, VTV Prime)
- - Priority: ssaimh CDN → sv2.xemtivitop.com → xemtv.us → xemtivitop blogspot → FPTPlay → VTVGo → mediacdn → xemtv.net
-- 🏆 World Cup 2026 (news, fixtures, standings, stats, highlights)
-- 🤖 AI article writing + TTS (multilingual, emotion-aware)
-- 🔍 Topic search (8 news sources)
-- 🎤 TTS: voice selector + emotion selector + speed control
-
-## 🎬 Short AI — Video từ link (scrap YouTube / TikTok / tin tức)
-
-Short creator có chế độ **"🔗 Video từ link"**: dán link video YouTube / TikTok /
-VnExpress / Dân trí / Znews / 24h... → bấm "Lấy video" để xem trước → tạo short
-chạy video + ảnh đã chọn bù phần còn thiếu nếu video ngắn hơn giọng đọc.
-
-### 🔑 Cài cookies cho YouTube (bỏ chặn "Sign in to confirm you're not a bot")
-YouTube đôi khi chặn IP datacenter. Cách khắc phục bằng cookies:
-
-1. Cài extension trình duyệt **"Get cookies.txt LOCALLY"** (Chrome/Edge) hoặc
- **"cookies.txt"** (Firefox).
-2. Mở `https://www.youtube.com` (đã đăng nhập) → bấm extension → **Export** → ra file `cookies.txt` (định dạng Netscape).
-3. Đưa cookies vào Space bằng **một trong hai cách**:
- - **Cách A (khuyến nghị):** Vào Settings của Space
- `huggingface.co/spaces/bep40/VNEWS/settings` → **Variables and secrets** →
- tạo secret tên **`YT_COOKIES`**, giá trị = toàn bộ nội dung file `cookies.txt`.
- - **Cách B:** đặt file `cookies.txt` vào thư mục gốc repo `VNEWS/` và commit
- (chú ý: cookies sẽ công khai nếu repo public — ưu tiên Cách A).
-4. Rebuild Space (mỗi lần đổi secret phải **Restart** Space).
-
-Backend tự đọc `YT_COOKIES` (secret) hoặc `/app/cookies.txt`, ghi thành file tạm
-và truyền cho yt-dlp qua `cookiefile`. Không cần sửa code.
-
-> Lưu ý: cookies có hạn (thường vài tuần). Khi hết hạn, export lại và cập nhật secret.
\ No newline at end of file
+# bep40/vnews
+
diff --git a/RESTART_TRIGGER.md b/RESTART_TRIGGER.md
deleted file mode 100644
index e6c79c75d2068b3dc20f52912afac9b21944b2d0..0000000000000000000000000000000000000000
--- a/RESTART_TRIGGER.md
+++ /dev/null
@@ -1,7 +0,0 @@
-trigger rebuild v20260827fptwall
-
-- FPT wall cards now use .wall-thumb 100% width/height (16:9), identical to Short AI cards
-- FPT player opens a scrollable vertical feed interleaving FPT Play + Short AI videos, newest-first
-- 16:9 <-> 9:16 ratio toggle (🖥️/📺) works on football highlight slides and Tường AI slide player
-- Cache-busting: app_v2.js?v=20260827fptwall
-- FORCE REBUILD: this commit retriggers the Space container with the latest static/app_v2.js
\ No newline at end of file
diff --git a/_run.py b/_run.py
deleted file mode 100644
index de72380c3006dba490ec49c1fe421c87723a0490..0000000000000000000000000000000000000000
--- a/_run.py
+++ /dev/null
@@ -1 +0,0 @@
-from app_v2_entry import app # v5-stable inline bongda proxy
\ No newline at end of file
diff --git a/ai_ext.py b/ai_ext.py
index ab7540db0765711cf34947d481888084e0f87c65..9ddd41c6072ff2a9e13fd503618277a7753f569e 100644
--- a/ai_ext.py
+++ b/ai_ext.py
@@ -13,16 +13,7 @@ from bs4 import BeautifulSoup
from fastapi import Request, Query
from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
-# Try to import main app, but don't fail if it doesn't exist
-try:
- from main import app
-except ImportError:
- # Create a minimal FastAPI app for standalone testing
- try:
- from fastapi import FastAPI
- app = FastAPI()
- except Exception:
- app = None
+from main import app
# Import wall store from main.py so we read/write the SAME file
try:
@@ -50,10 +41,6 @@ except ImportError:
def _web_context(topic):
return ""
-# ai_ext alias for backward compatibility
-_load_ai_wall = _load_wall
-_save_ai_wall = _save_wall
-
try:
from huggingface_hub import AsyncInferenceClient
except Exception:
@@ -73,260 +60,994 @@ except Exception:
def _hf_token():
- for k in ("HF_TOKEN", "HUGGINGFACE_HUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_API_TOKEN"):
+ for k in ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_API_TOKEN"):
v = os.getenv(k, "").strip()
if v:
return v
return ""
+HF_TOKEN = _hf_token()
+QWEN_VL_MODEL = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
+DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
+SHORTS_DIR = os.path.join(DATA_DIR, "ai_shorts")
+HEADERS = {
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+ "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"
+}
+LAST_QWEN_ERROR = ""
+
+# ===== TTS VOICE CONFIG =====
+TTS_VOICES = {
+ "hoaimy": {"id": "vi-VN-HoaiMyNeural", "gender": "female", "name": "Hoài My", "country": "Việt Nam", "lang": "vi", "flag": "🇻🇳"},
+ "namminh": {"id": "vi-VN-NamMinhNeural", "gender": "male", "name": "Nam Minh", "country": "Việt Nam", "lang": "vi", "flag": "🇻🇳"},
+ "en_us_f": {"id": "en-US-AvaNeural", "gender": "female", "name": "Ava", "country": "Mỹ", "lang": "en", "flag": "🇺🇸"},
+ "en_us_m": {"id": "en-US-BrianNeural", "gender": "male", "name": "Brian", "country": "Mỹ", "lang": "en", "flag": "🇺🇸"},
+ "en_gb_f": {"id": "en-GB-SoniaNeural", "gender": "female", "name": "Sonia", "country": "Anh", "lang": "en", "flag": "🇬🇧"},
+ "en_gb_m": {"id": "en-GB-RyanNeural", "gender": "male", "name": "Ryan", "country": "Anh", "lang": "en", "flag": "🇬🇧"},
+ "zh_cn_f": {"id": "zh-CN-XiaoxiaoNeural", "gender": "female", "name": "Xiaoxiao", "country": "Trung Quốc", "lang": "zh", "flag": "🇨🇳"},
+ "zh_cn_m": {"id": "zh-CN-YunyangNeural", "gender": "male", "name": "Yunyang", "country": "Trung Quốc", "lang": "zh", "flag": "🇨🇳"},
+ "zh_hk_f": {"id": "zh-HK-HiuGaaiNeural", "gender": "female", "name": "HiuGaai", "country": "Hồng Kông", "lang": "zh", "flag": "🇭🇰"},
+ "ja_jp_f": {"id": "ja-JP-NanamiNeural", "gender": "female", "name": "Nanami", "country": "Nhật Bản", "lang": "ja", "flag": "🇯🇵"},
+ "ja_jp_m": {"id": "ja-JP-KeitaNeural", "gender": "male", "name": "Keita", "country": "Nhật Bản", "lang": "ja", "flag": "🇯🇵"},
+ "ko_kr_f": {"id": "ko-KR-SunHiNeural", "gender": "female", "name": "Sun-Hi", "country": "Hàn Quốc", "lang": "ko", "flag": "🇰🇷"},
+ "ko_kr_m": {"id": "ko-KR-InJoonNeural", "gender": "male", "name": "InJoon", "country": "Hàn Quốc", "lang": "ko", "flag": "🇰🇷"},
+ "fr_fr_f": {"id": "fr-FR-DeniseNeural", "gender": "female", "name": "Denise", "country": "Pháp", "lang": "fr", "flag": "🇫🇷"},
+ "fr_fr_m": {"id": "fr-FR-HenriNeural", "gender": "male", "name": "Henri", "country": "Pháp", "lang": "fr", "flag": "🇫🇷"},
+ "de_de_f": {"id": "de-DE-KatjaNeural", "gender": "female", "name": "Katja", "country": "Đức", "lang": "de", "flag": "🇩🇪"},
+ "de_de_m": {"id": "de-DE-KillianNeural", "gender": "male", "name": "Killian", "country": "Đức", "lang": "de", "flag": "🇩🇪"},
+ "es_es_f": {"id": "es-ES-XimenaNeural", "gender": "female", "name": "Ximena", "country": "Tây Ban Nha", "lang": "es", "flag": "🇪🇸"},
+ "es_mx_m": {"id": "es-MX-JorgeNeural", "gender": "male", "name": "Jorge", "country": "Mexico", "lang": "es", "flag": "🇲🇽"},
+ "th_th_f": {"id": "th-TH-PremwadeeNeural", "gender": "female", "name": "Premwadee", "country": "Thái Lan", "lang": "th", "flag": "🇹🇭"},
+ "th_th_m": {"id": "th-TH-NiwatNeural", "gender": "male", "name": "Niwat", "country": "Thái Lan", "lang": "th", "flag": "🇹🇭"},
+ "hi_in_f": {"id": "hi-IN-SwaraNeural", "gender": "female", "name": "Swara", "country": "Ấn Độ", "lang": "hi", "flag": "🇮🇳"},
+ "hi_in_m": {"id": "hi-IN-MadhurNeural", "gender": "male", "name": "Madhur", "country": "Ấn Độ", "lang": "hi", "flag": "🇮🇳"},
+}
+TTS_DEFAULT_VOICE = "hoaimy"
+TTS_DEFAULT_SPEED = 1.2
+
+TOPIC_VOICE_MAP = {
+ "bóng đá": "namminh", "thể thao": "namminh", "world cup": "namminh",
+ "premier league": "namminh", "champions league": "namminh", "la liga": "namminh",
+ "serie a": "namminh", "bundesliga": "namminh", "v-league": "namminh",
+ "tennis": "namminh", "olympic": "namminh", "f1": "namminh", "moto": "namminh",
+ "sức khỏe": "hoaimy", "làm đẹp": "hoaimy", "giải trí": "hoaimy",
+ "âm nhạc": "hoaimy", "phim": "hoaimy", "thời trang": "hoaimy",
+ "ẩm thực": "hoaimy", "du lịch": "hoaimy", "gia đình": "hoaimy",
+ "tình yêu": "hoaimy", "hôn nhân": "hoaimy", "mẹ và bé": "hoaimy",
+ "công nghệ": "namminh", "ai": "namminh", "robot": "namminh",
+ "khoa học": "namminh", "vũ trụ": "namminh", "điện thoại": "namminh",
+ "laptop": "namminh", "game": "namminh",
+ "chính trị": "namminh", "kinh tế": "namminh", "tài chính": "namminh",
+ "chứng khoán": "namminh", "ngân hàng": "namminh", "thị trường": "namminh",
+ "xã hội": "namminh", "pháp luật": "namminh", "giáo dục": "namminh",
+}
+
+
+def _detect_voice_for_topic(title: str, text: str) -> str:
+ combined = (title + " " + text[:500]).lower()
+ for keyword, voice_id in TOPIC_VOICE_MAP.items():
+ if keyword in combined:
+ return voice_id
+ return TTS_DEFAULT_VOICE
+
+# ===== TEXT HELPERS =====
def _clean_text(s: str) -> str:
- """Clean text for processing."""
s = html_lib.unescape(s or "")
- s = re.sub(r"\s+", " ", s)
- return s.strip()
+ return re.sub(r"\s+", " ", s).strip()
-
-def _domain(url: str) -> str:
- """Extract domain from URL."""
+def _domain(u):
try:
- return urlparse(url or "").netloc.replace("www.", "")
+ return urlparse(u).netloc.replace("www.", "")
except Exception:
return ""
+def _safe_name(s):
+ return re.sub(r"[^a-zA-Z0-9_-]+", "_", str(s))[:80]
-async def qwen_generate(prompt: str, image_url: str = None, max_tokens: int = 1200) -> str:
- """Generate text using Llama/Qwen models via Hugging Face Inference API.
-
- Prioritizes Llama-3.3-70B for better creative/opinion writing.
+
+# ===== CLEAN AI OUTPUT — COMPLETE REWRITE =====
+def _clean_ai_output(text: str) -> str:
"""
- token = _hf_token()
- errors = []
-
- # Try HF router API with multiple models - Llama FIRST for opinion writing
- if token:
- models = [
- os.getenv("QWEN_VL_MODEL", ""),
- "meta-llama/Llama-3.3-70B-Instruct", # FIRST - best for opinion/analysis
- "Qwen/Qwen2.5-VL-7B-Instruct",
- "Qwen/Qwen2.5-72B-Instruct",
- ]
- # Deduplicate while preserving order
- seen = set()
- models = [m for m in models if m and m not in seen and not seen.add(m)]
-
- headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
-
- for model in models:
- try:
- is_vl = "VL" in model and image_url
- if is_vl:
- user_content = [
- {"type": "image_url", "image_url": {"url": image_url}},
- {"type": "text", "text": prompt}
- ]
- else:
- user_content = prompt
-
- payload = {
- "model": model,
- "messages": [
- {"role": "system", "content": "Bạn là nhà báo phản biện chuyên nghiệp. Luôn viết theo quan điểm cá nhân, phân tích sâu, không sao chép nguyên văn nguồn tin."},
- {"role": "user", "content": user_content},
- ],
- "max_tokens": min(int(max_tokens or 2000), 2500),
- "temperature": 0.75,
- "top_p": 0.9,
- }
-
- r = requests.post(
- "https://router.huggingface.co/v1/chat/completions",
- headers=headers,
- json=payload,
- timeout=95
- )
-
- if r.status_code >= 300:
- errors.append(f"{model}: HTTP {r.status_code}")
- continue
-
- j = r.json()
- txt = (j.get("choices", [{}])[0].get("message", {}).get("content") or "").strip()
-
- if txt:
- return txt
-
- errors.append(f"{model}: empty response")
-
- except Exception as e:
- errors.append(f"{model}: {type(e).__name__}")
-
- # Fallback: extractive summary from prompt
- LAST_QWEN_ERROR = errors[-3:] if errors else "unknown error"
- return _fallback_summary_from_prompt(prompt, max_units=6)
+ Aggressively clean AI output:
+ 1. Remove markdown artifacts
+ 2. Remove instruction leakage / meta-commentary
+ 3. Split into sentences, deduplicate (exact + near-match)
+ 4. Trim at complete sentence boundary (no mid-sentence cuts)
+ 5. Remove consecutive duplicate lines
+ """
+ if not text:
+ return ""
+ # --- Step 1: Remove markdown ---
+ text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
+ text = re.sub(r'\*([^*]+)\*', r'\1', text)
+ text = re.sub(r'^---+\s*$', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^[-*_]{3,}\s*$', '', text, flags=re.MULTILINE)
-def _fallback_summary_from_prompt(prompt: str, max_units: int = 6) -> str:
- """Generate a simple fallback summary when AI is unavailable."""
- text = prompt or ""
- for marker in ["Nội dung nguồn:", "Nội dung bài:", "Nội dung gốc:", "Nội dung:", "Nguồn/bối cảnh internet:"]:
- if marker in text:
- text = text.split(marker, 1)[1]
- break
- text = re.sub(r"https?://\S+", "", text)
- text = re.sub(r"\s+", " ", text).strip()
-
- # Split into sentences
- sentences = re.split(r"(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])", text)
- units = []
- for s in sentences:
- s = _clean_text(s)
- if len(s) >= 30:
- units.append(s)
-
- if units:
- result_units = units[:max_units]
- return "\n".join("• " + u for u in result_units)
- if text:
- chunks = []
- for i in range(0, min(len(text), max_units * 300), 280):
- chunk = _clean_text(text[i:i+300])
- if chunk and chunk not in chunks:
- chunks.append(chunk)
- if len(chunks) >= max_units:
+ # --- Step 2: Remove instruction leakage (entire lines) ---
+ leakage_patterns = [
+ r'Dưới đây là', r'Theo yêu cầu', r'Tôi sẽ viết', r'Tôi sẽ tóm tắt',
+ r'Đây là bài', r'Đây là nội dung', r'Bài viết sau đây',
+ r'Nội dung (tóm tắt|chính)', r'Nhiệm vụ', r'Vai trò', r'Tôi là',
+ r'Dựa trên.*tôi sẽ', r'Hãy', r'Bạn cần', r'Đọc bài viết',
+ r'Tôi xin', r'Xin chào', r'Trân trọng', r'Kính thưa',
+ r'Dựa trên.*dưới đây', r'Sau đây là', r'Dưới đây là bài',
+ r'Dựa trên.*tôi đã', r'Theo nội dung', r'Tóm tắt như sau',
+ r'Dưới đây là bài viết', r'Đây là phần', r'Bài tóm tắt',
+ r'Nội dung tóm tắt', r'Đoạn văn sau', r'Viết lại.*sau',
+ ]
+ for phrase in leakage_patterns:
+ text = re.sub(r'^[^\n]*' + phrase + r'[^\n]*\n?', '', text, flags=re.MULTILINE | re.IGNORECASE)
+
+ # Remove lines that are just meta-instructions
+ text = re.sub(r'^[^\n]*(?:QUY TẮC|bắt buộc|tối đa|không dùng)[^\n]*\n?', '', text, flags=re.MULTILINE | re.IGNORECASE)
+
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = text.strip()
+
+ if not text:
+ return ""
+
+ # --- Step 3: Split into sentences & deduplicate ---
+ # Vietnamese sentences end with . ! ? or end of string
+ # We split on sentence-ending punctuation followed by space or end
+ raw_parts = re.split(r'(?<=[.!?])\s+', text.strip())
+
+ def _norm(s):
+ """Normalize for comparison: lowercase, collapse whitespace, remove punctuation."""
+ s = re.sub(r'\s+', ' ', s.strip().lower())
+ return re.sub(r'[^\w\s]', '', s)
+
+ seen_sentences = set()
+ unique_parts = []
+
+ for part in raw_parts:
+ n = _norm(part)
+ if not n or len(n) < 5:
+ # Very short fragments — keep them if they're not empty
+ if part.strip():
+ unique_parts.append(part)
+ continue
+
+ # Check exact match
+ if n in seen_sentences:
+ continue
+
+ # Check near-duplicate: substring match
+ is_dup = False
+ n_clean = re.sub(r'\s+', '', n)
+ for seen in list(seen_sentences):
+ seen_clean = re.sub(r'\s+', '', seen)
+ if not n_clean or not seen_clean:
+ continue
+ # One is substring of the other
+ if n_clean in seen_clean or seen_clean in n_clean:
+ shorter = min(len(n_clean), len(seen_clean))
+ longer = max(len(n_clean), len(seen_clean))
+ if shorter > 15 and shorter / longer > 0.7:
+ is_dup = True
break
- if chunks:
- return "\n".join("• " + c for c in chunks)
- return "• Không có đủ nội dung để tóm tắt."
+ # High word overlap
+ else:
+ words_n = set(n.split())
+ words_s = set(seen.split())
+ if len(words_n) >= 3 and len(words_s) >= 3:
+ overlap = len(words_n & words_s)
+ ratio = overlap / min(len(words_n), len(words_s))
+ if ratio > 0.75:
+ is_dup = True
+ break
-# ===== URL scraping & article processing =====
-HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"}
+ if is_dup:
+ continue
-try:
- _shorts_base = "/data" if os.path.isdir("/data") else os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
-except Exception:
- _shorts_base = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
-SHORTS_DIR = os.path.join(_shorts_base, "ai_shorts")
-os.makedirs(SHORTS_DIR, exist_ok=True)
+ seen_sentences.add(n)
+ unique_parts.append(part)
-import random as _random2
-from datetime import datetime, timezone, timedelta
-_VN_TZ = timezone(timedelta(hours=7))
+ result = ' '.join(unique_parts).strip()
+ # --- Step 4: Remove consecutive duplicate lines ---
+ lines = result.split('\n')
+ final_lines = []
+ prev_line = ""
+ for line in lines:
+ stripped = line.strip()
+ if stripped and stripped == prev_line:
+ continue
+ final_lines.append(line)
+ prev_line = stripped
+ result = '\n'.join(final_lines).strip()
-def _safe_name(filename: str) -> str:
- """Sanitize filename."""
- return re.sub(r"[^a-zA-Z0-9_.-]", "_", filename)[:120]
+ # --- Step 5: Trim at complete sentence boundary ---
+ # Find the last sentence-ending punctuation and cut there
+ # This prevents mid-sentence truncation
+ if result:
+ # Find all sentence-ending positions
+ end_matches = list(re.finditer(r'[.!?]\s*$', result))
+ if not end_matches:
+ # No ending punctuation — try to find the last complete sentence
+ last_end = -1
+ for m in re.finditer(r'[.!?](?=\s|$)', result):
+ last_end = m.end()
+ if last_end > 0:
+ result = result[:last_end].strip()
+ # else: already ends with punctuation, good
+ # --- Step 6: Final cleanup ---
+ # Remove any trailing incomplete sentence fragments after last punctuation
+ # e.g. "Đây là câu 1. Đây là câu 2. Và một" → cut "Và một"
+ last_period = result.rfind('.')
+ last_exclaim = result.rfind('!')
+ last_question = result.rfind('?')
+ last_end = max(last_period, last_exclaim, last_question)
+ if last_end > 0:
+ after = result[last_end + 1:].strip()
+ # If what follows the last sentence end is a short fragment (< 20 chars, no ending punct), cut it
+ if after and len(after) < 20 and not re.search(r'[.!?]$', after):
+ result = result[:last_end + 1].strip()
-def pollinations_image_url(topic: str) -> str:
- """Generate a placeholder image URL via Pollinations."""
+ return result
+
+
+# ===== EXTRACT ALL IMAGES FROM ARTICLE — ENHANCED =====
+def _extract_all_images(soup, base_url: str) -> List[Dict]:
+ """Extract ALL content images from an article page using multi-strategy approach."""
+ images = []
+ seen_urls = set()
+ skip_patterns = [
+ "avatar", "icon", "logo", "button", "banner-ad", "tracking",
+ "beacon", "pixel", "1x1", "spacer", "emoji", "sprite", "placeholder",
+ "advertisement", "ads", "widget", "sidebar", "footer-logo",
+ "social", "share", "comment-avatar", "user-avatar",
+ ]
+
+ def _add_image(src: str, alt: str = "", source_tag: str = "img"):
+ if not src or src.startswith("data:"):
+ return
+ # Normalize URL
+ src = src.strip()
+ if src.startswith("//"):
+ src = "https:" + src
+ abs_url = urljoin(base_url, src)
+ if abs_url in seen_urls:
+ return
+ # Skip non-content images by URL pattern
+ if any(p in abs_url.lower() for p in skip_patterns):
+ return
+ # Skip very small images (likely icons)
+ try:
+ parsed = urlparse(abs_url)
+ path = parsed.path.lower()
+ if any(path.endswith(ext) for ext in ['.svg', '.ico']):
+ return
+ # Skip if URL contains dimension patterns like 16x16, 32x32
+ if re.search(r'/\d{1,3}x\d{1,3}/', path) or re.search(r'-\d{1,3}x\d{1,3}\.', path):
+ return
+ except Exception:
+ pass
+ seen_urls.add(abs_url)
+ images.append({"url": abs_url, "alt": alt, "source": source_tag})
+
+ # Strategy 1: Standard with
+ for picture in soup.find_all("picture"):
+ for source in picture.find_all("source"):
+ srcset = source.get("srcset", "")
+ for part in srcset.split(","):
+ part = part.strip()
+ if part:
+ _add_image(part.split(" ")[0], source_tag="picture/srcset")
+ fallback_img = picture.find("img")
+ if fallback_img:
+ _add_image(
+ fallback_img.get("src") or fallback_img.get("data-src"),
+ alt=fallback_img.get("alt", ""),
+ source_tag="picture/img"
+ )
+
+ # Strategy 4: WordPress CMS patterns
+ for img in soup.find_all("img", class_=re.compile(r"wp-image|size-large|size-full|aligncenter")):
+ _add_image(img.get("data-src") or img.get("src"),
+ alt=img.get("alt", ""), source_tag="wp-image")
+
+ # Strategy 5: Background images in style attributes
+ for tag in soup.find_all(style=re.compile(r"background-image")):
+ for m in re.findall(r'url\(["\']?(.*?)["\']?\)', tag.get("style", "")):
+ _add_image(m, source_tag="background-style")
+
+ # Strategy 6: og:image (featured/hero image)
+ og_image = soup.find("meta", property="og:image")
+ if og_image and og_image.get("content"):
+ _add_image(og_image["content"], source_tag="og:image")
+
+ # Strategy 7: twitter:image
+ tw_image = soup.find("meta", attrs={"name": "twitter:image"})
+ if tw_image and tw_image.get("content"):
+ _add_image(tw_image["content"], source_tag="twitter:image")
+
+ # Strategy 8: with
+ for figure in soup.find_all("figure"):
+ img = figure.find("img")
+ if img:
+ src = img.get("data-src") or img.get("src")
+ figcaption = figure.find("figcaption")
+ alt = figcaption.get_text(strip=True) if figcaption else img.get("alt", "")
+ _add_image(src, alt=alt, source_tag="figure")
+
+ # Strategy 9: tags linking to images
+ for a in soup.find_all("a", href=True):
+ href = a["href"]
+ if any(href.lower().endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".webp"]):
+ _add_image(href, alt=a.get_text(strip=True)[:80], source_tag="link")
+
+ # Strategy 10: VnExpress-specific — fig-parent / fig-image containers
+ for fig_parent in soup.find_all(class_=re.compile(r"fig-parent|fig-image|image|thumb")):
+ img = fig_parent.find("img")
+ if img:
+ src = img.get("data-src") or img.get("src") or img.get("data-original")
+ _add_image(src, alt=img.get("alt", ""), source_tag="fig-parent")
+ # Also check for background-image in fig-parent
+ style = fig_parent.get("style", "")
+ for m in re.findall(r'url\(["\']?(.*?)["\']?\)', style):
+ _add_image(m, source_tag="fig-parent-bg")
+
+ # Strategy 11: Dân Trí / VietnamNet — image in .content-detail, .article-body
+ for container in soup.find_all(class_=re.compile(r"content-detail|article-body|article_content|detail-content|singular-content|fck_detail|knc-content|cms-body|article__body")):
+ for img in container.find_all("img"):
+ src = img.get("data-src") or img.get("src") or img.get("data-original")
+ _add_image(src, alt=img.get("alt", ""), source_tag="content-block-img")
+
+ # Strategy 12: Generic — any or
+ for scope in soup.find_all(["article", "main"]):
+ for img in scope.find_all("img"):
+ src = img.get("data-src") or img.get("src") or img.get("data-original")
+ _add_image(src, alt=img.get("alt", ""), source_tag="article-scope")
+
+ return images
+
+
+# ===== JINA READER =====
+def _reader_url(target_url: str) -> str:
+ safe = quote(target_url, safe=":/?#[]@!$&'()*+,;=%")
+ return "https://r.jina.ai/http://" + safe
+
+def jina_reader_markdown(url: str) -> str:
+ jr = _reader_url(url)
+ r = requests.get(jr, headers={"Accept": "text/markdown,text/plain,*/*", "X-Return-Format": "markdown", "User-Agent": "Mozilla/5.0"}, timeout=35)
+ r.raise_for_status()
+ return r.text or ""
+
+def _parse_jina_markdown(md: str, url: str):
+ lines = [x.rstrip() for x in (md or "").splitlines()]
+ title = ""; first_image = ""; all_images = []; content_lines = []; in_content = False
+ for ln in lines:
+ if ln.startswith("Title:") and not title:
+ title = _clean_text(ln.replace("Title:", "", 1)); continue
+ if ln.startswith("URL Source:"):
+ continue
+ if ln.startswith("Markdown Content:"):
+ in_content = True; continue
+ # Extract ALL images from markdown 
+ for mimg in re.finditer(r'!\[[^\]]*\]\((https?://[^)]+)\)', ln):
+ img_url = mimg.group(1)
+ if img_url not in all_images:
+ all_images.append(img_url)
+ if not first_image:
+ first_image = img_url
+ if in_content or (title and not ln.startswith("Title:")):
+ if ln.strip():
+ content_lines.append(ln)
+ text = "\n".join(content_lines)
+ text = re.sub(r'!\[[^\]]*\]\([^)]+\)', '', text)
+ paras = []
+ for part in re.split(r'\n{2,}|\n(?=#{1,3}\s)', text):
+ t = _clean_text(re.sub(r'^#{1,6}\s*', '', part))
+ if len(t) >= 40:
+ paras.append(t)
+ if len(paras) >= 35:
+ break
+ if not title and paras:
+ title = paras[0][:90]
+ return {"url": url, "title": title or url, "summary": paras[0] if paras else "",
+ "text": "\n".join(paras), "image": first_image,
+ "images": all_images, "via": "jina"}
+
+
+# ===== WEB SCRAPE (with full image extraction) =====
+def _best_content_block(soup):
+ best, best_score = None, 0
+ for el in soup.find_all(["article", "main", "section", "div"]):
+ ps = el.find_all("p")
+ txt = " ".join(p.get_text(" ", strip=True) for p in ps)
+ score = len(ps) * 100 + len(txt)
+ cls = " ".join(el.get("class", []))
+ if any(k in cls.lower() for k in ["content", "article", "detail", "body", "post", "entry"]):
+ score += 800
+ if score > best_score:
+ best, best_score = el, score
+ return best
+
+def scrape_any_url_direct(url: str):
+ r = requests.get(url, headers=HEADERS, timeout=18)
+ if r.status_code in {401, 403, 406, 409, 429, 451, 503}:
+ raise RuntimeError(f"blocked status {r.status_code}")
+ r.encoding = "utf-8"
+ soup = BeautifulSoup(r.text, "lxml")
+ for tag in soup.find_all(["script", "style", "nav", "footer", "aside", "form", "noscript"]):
+ tag.decompose()
+
+ # Title
+ title = soup.find("h1").get_text(" ", strip=True) if soup.find("h1") else ""
+ if not title:
+ ogt = soup.find("meta", property="og:title") or soup.find("meta", attrs={"name": "title"})
+ title = ogt.get("content", "") if ogt else (soup.title.get_text(strip=True) if soup.title else "")
+
+ # Summary
+ desc_tag = soup.find("meta", property="og:description") or soup.find("meta", attrs={"name": "description"})
+ summary = desc_tag.get("content", "") if desc_tag else ""
+
+ # Featured image (og:image)
+ img_tag = soup.find("meta", property="og:image") or soup.find("meta", attrs={"name": "twitter:image"})
+ image = img_tag.get("content", "") if img_tag else ""
+ if image and image.startswith("//"):
+ image = "https:" + image
+
+ # Extract ALL images from the article
+ all_images = _extract_all_images(soup, url)
+ image_urls = [img["url"] for img in all_images]
+
+ # Ensure featured image is first
+ if image and image not in image_urls:
+ image_urls.insert(0, image)
+ elif image in image_urls:
+ image_urls.remove(image)
+ image_urls.insert(0, image)
+
+ # Content paragraphs
+ block = _best_content_block(soup) or soup
+ paras, seen_p = [], set()
+ for p in block.find_all("p"):
+ t = _clean_text(p.get_text(" ", strip=True))
+ if len(t) >= 40 and t not in seen_p:
+ seen_p.add(t)
+ paras.append(t)
+ if len(paras) >= 35:
+ break
+
+ if not title and paras:
+ title = paras[0][:90]
+
+ return {
+ "url": url, "title": title or url, "summary": paras[0] if paras else "",
+ "text": "\n".join(paras), "image": image_urls[0] if image_urls else "",
+ "images": image_urls, "via": _domain(url)
+ }
+
+def scrape_any_url(url: str):
+ """Try direct scrape first, fall back to Jina Reader.
+
+ KEY FIX: When falling back to Jina, MERGE images from both sources
+ instead of overwriting. Direct scrape often finds more inline images.
+ """
+ data = scrape_any_url_direct(url)
+ raw_text = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
+ direct_images = data.get("images", [])
+
+ if len(raw_text) >= 120:
+ # Direct scrape got enough content — but still try Jina for more images
+ # Only use Jina if direct scrape found very few images
+ if len(direct_images) < 3:
+ try:
+ md = jina_reader_markdown(url)
+ if md:
+ jr = _parse_jina_markdown(md, url)
+ if jr.get("images"):
+ # MERGE: add Jina images that aren't already in direct scrape
+ existing = set(direct_images)
+ for img_url in jr["images"]:
+ if img_url not in existing:
+ direct_images.append(img_url)
+ existing.add(img_url)
+ data["images"] = direct_images
+ except Exception:
+ pass
+ return data
+
+ # Direct scrape didn't get enough content — fall back to Jina
try:
- return "https://image.pollinations.ai/prompt/" + quote("Vietnamese editorial illustration, " + topic, safe="") + "?width=1024&height=576&nologo=true"
+ md = jina_reader_markdown(url)
+ if md:
+ jr = _parse_jina_markdown(md, url)
+ if jr.get("text"):
+ # Keep direct scrape's title/images if Jina's are worse
+ if data.get("title") and data["title"] != url:
+ jr["title"] = data["title"]
+ # MERGE images: direct scrape images + Jina images
+ merged_images = list(direct_images) # start with direct scrape images
+ existing = set(merged_images)
+ for img_url in jr.get("images", []):
+ if img_url not in existing:
+ merged_images.append(img_url)
+ existing.add(img_url)
+ jr["images"] = merged_images
+ # Use direct scrape's image as first if available
+ if data.get("image"):
+ jr["image"] = data["image"]
+ jr["via"] = data.get("via", _domain(url)) + " + jina"
+ return jr
except Exception:
- return ""
+ pass
+ return data
+
+
+# ===== POLLINATIONS IMAGE =====
+def pollinations_image_url(topic: str) -> str:
+ prompt = "editorial illustration, Vietnamese news, " + topic
+ return "https://image.pollinations.ai/prompt/" + quote(prompt, safe="") + "?width=1024&height=576&nologo=true"
+
+
+# ===== QWEN AI (strict, concise) =====
+async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 700, image_urls: Optional[List[str]] = None):
+ global LAST_QWEN_ERROR, HF_TOKEN
+ HF_TOKEN = _hf_token()
+ if not HF_TOKEN:
+ LAST_QWEN_ERROR = "Không tìm thấy token"
+ return None
+ if not AsyncInferenceClient:
+ LAST_QWEN_ERROR = "Thiếu huggingface_hub"
+ return None
+ errors = []; models = []
+ for m in [QWEN_VL_MODEL, "Qwen/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]:
+ if m and m not in models:
+ models.append(m)
+ for model in models:
+ try:
+ client = AsyncInferenceClient(provider="auto", api_key=HF_TOKEN, timeout=90)
+ content = []
+ # Collect all images: image_urls list takes priority, fall back to single image_url
+ all_img_urls = []
+ if image_urls:
+ all_img_urls = image_urls[:6] # max 6 images to avoid context overflow
+ elif image_url:
+ all_img_urls = [image_url]
+ for img_u in all_img_urls:
+ if img_u and img_u.startswith("http"):
+ content.append({"type": "image_url", "image_url": {"url": img_u}})
+ content.append({"type": "text", "text": prompt})
+ messages = [
+ {"role": "system", "content": (
+ "Bạn là biên tập viên báo điện tử tiếng Việt. "
+ "NHIỆM VỤ: Chỉ TÓM TẮT nội dung, KHÔNG viết lại bài đầy đủ. "
+ "QUY TẮC CỨNG: "
+ "(1) KHÔNG lặp lại bất kỳ nội dung nào — mỗi ý chỉ xuất hiện ĐÚNG 1 LẦN. "
+ "(2) Nếu 2 câu diễn đạt cùng 1 ý → bỏ cây thứ 2. "
+ "(3) KHÔNG dùng Markdown (##, **, ---, *). "
+ "(4) KHÔNG viết 'Dưới đây là', 'Tôi sẽ', 'Theo yêu cầu', 'Nhiệm vụ', 'Vai trò', 'Đây là bài tóm tắt'. "
+ "(5) KHÔNG bịa thông tin ngoài nguồn. "
+ "(6) Chỉ viết ĐOẠN VĂN THUẦN, không bullet points. "
+ "(7) Tối đa 200 từ. Ngắn gọn, súc tích. "
+ "(8) KHÔNG sao chép nguyên văn từ bài gốc. Viết lại bằng cách diễn đạt khác hoàn toàn."
+ )},
+ {"role": "user", "content": content}
+ ]
+ resp = await client.chat_completion(model=model, messages=messages, max_tokens=max_tokens, temperature=0.3, top_p=0.8)
+ txt = (resp.choices[0].message.content or "").strip()
+ if txt:
+ LAST_QWEN_ERROR = ""
+ return txt
+ except Exception as e:
+ errors.append(f"{model}: {type(e).__name__}: {str(e)[:220]}")
+ LAST_QWEN_ERROR = " | ".join(errors) or "Qwen không trả nội dung."
+ print("[qwen errors]", LAST_QWEN_ERROR)
+ return None
+
+
+# ===== TTS GENERATION =====
+async def _generate_tts_edge(text: str, voice_id: str, speed: float, out_path: str):
+ if edge_tts is None:
+ raise RuntimeError("edge-tts chưa cài đặt")
+ voice = TTS_VOICES.get(voice_id, TTS_VOICES[TTS_DEFAULT_VOICE])["id"]
+ pct = int((speed - 1.0) * 100)
+ rate = f"+{pct}%" if pct >= 0 else f"{pct}%"
+ communicate = edge_tts.Communicate(text, voice, rate=rate)
+ await communicate.save(out_path)
+
+def _generate_tts_gtts(text: str, out_path: str):
+ if gTTS is None:
+ raise RuntimeError("gTTS chưa cài đặt")
+ gTTS(text, lang="vi").save(out_path)
-def _download_image(url: str, fallback_title: str, out_path: str) -> str:
- """Download an image from URL or create a placeholder."""
+# ===== SHORT VIDEO GENERATION =====
+def _download_image(url, fallback_topic, out_path):
if url:
try:
r = requests.get(url, headers=HEADERS, timeout=15)
- if r.status_code == 200 and len(r.content) > 1200:
- os.makedirs(os.path.dirname(out_path), exist_ok=True)
+ if r.status_code == 200 and len(r.content) > 1000:
with open(out_path, "wb") as f:
f.write(r.content)
return out_path
except Exception:
pass
- # Fallback: create a placeholder image
+ gen = pollinations_image_url(fallback_topic)
try:
- from PIL import Image, ImageDraw, ImageFont
- img = Image.new("RGB", (1080, 760), (24, 24, 24))
- draw = ImageDraw.Draw(img)
- try:
- font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 48)
- except Exception:
- font = None
- text = (fallback_title or "VNEWS")[:40]
- try:
- bbox = draw.textbbox((0, 0), text, font=font)
- tw = bbox[2] - bbox[0]
- except Exception:
- tw = len(text) * 24
- draw.text(((1080 - tw) // 2, 330), text, fill=(255, 255, 255), font=font)
- os.makedirs(os.path.dirname(out_path), exist_ok=True)
- img.save(out_path, quality=90)
- return out_path
+ r = requests.get(gen, headers=HEADERS, timeout=25)
+ if r.status_code == 200 and len(r.content) > 1000:
+ with open(out_path, "wb") as f:
+ f.write(r.content)
+ return out_path
except Exception:
+ pass
+ if Image:
+ Image.new("RGB", (1080, 860), (30, 55, 42)).save(out_path)
return out_path
+ raise RuntimeError("Không tạo được ảnh")
+def _make_short_frame(post, img_path, out_path):
+ if Image is None:
+ raise RuntimeError("Pillow chưa sẵn sàng")
+ W, H = 1080, 1920
+ bg = Image.new("RGB", (W, H), (14, 14, 14))
+ try:
+ im = Image.open(img_path).convert("RGB")
+ target = (1080, 860)
+ im_ratio = im.width / im.height
+ target_ratio = target[0] / target[1]
+ if im_ratio > target_ratio:
+ new_h = target[1]; new_w = int(new_h * im_ratio)
+ else:
+ new_w = target[0]; new_h = int(new_w / im_ratio)
+ im = im.resize((new_w, new_h))
+ left = (new_w - target[0]) // 2; top = (new_h - target[1]) // 2
+ im = im.crop((left, top, left + target[0], top + target[1]))
+ bg.paste(im, (0, 0))
+ except Exception:
+ pass
+ draw = ImageDraw.Draw(bg)
+ try:
+ font_title = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 52)
+ font_body = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 40)
+ font_label = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 32)
+ except Exception:
+ font_title = font_body = font_label = None
+ draw.rectangle((0, 780, W, H), fill=(14, 14, 14))
+ draw.text((54, 830), "VNEWS · Tường AI", fill=(92, 184, 122), font=font_label)
+ title = post.get("title", "")
+ words = title.split(); lines_t = []; cur = ""
+ for w in words:
+ if len(cur) + len(w) + 1 <= 24:
+ cur = (cur + " " + w).strip()
+ else:
+ if cur: lines_t.append(cur)
+ cur = w
+ if cur: lines_t.append(cur)
+ draw.multiline_text((54, 900), "\n".join(lines_t[:3]), fill=(255, 255, 255), font=font_title, spacing=10)
+ body_text = post.get("text", "")
+ words_b = body_text.split(); lines_b = []; cur_b = ""
+ for w in words_b:
+ if len(cur_b) + len(w) + 1 <= 34:
+ cur_b = (cur_b + " " + w).strip()
+ else:
+ if cur_b: lines_b.append(cur_b)
+ cur_b = w
+ if len(lines_b) >= 10:
+ break
+ if cur_b and len(lines_b) < 10: lines_b.append(cur_b)
+ draw.multiline_text((54, 1120), "\n".join(lines_b), fill=(220, 220, 220), font=font_body, spacing=12)
+ bg.save(out_path, quality=92)
-def scrape_any_url(url: str) -> dict:
- """Scrape article content from any URL."""
- if not url or not url.startswith("http"):
- return {"title": "", "text": "", "summary": "", "image": "", "og_image": "", "via": ""}
+def _short_script(post, max_chars=700):
+ txt = _clean_text(post.get("text", ""))
+ if len(txt) > max_chars:
+ # Trim at complete sentence
+ trimmed = txt[:max_chars]
+ last_end = -1
+ for m in re.finditer(r'[.!?](?=\s|$)', trimmed):
+ last_end = m.end()
+ if last_end > 0:
+ txt = trimmed[:last_end]
+ else:
+ txt = trimmed.rsplit(" ", 1)[0] + "."
+ title = _clean_text(post.get("title", ""))
+ return f"{title}. {txt}"
+
+async def _generate_short_video(post, post_id: str, voice_id: str = None, speed: float = None) -> str:
try:
- r = requests.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
- if r.status_code != 200 or not r.text:
- return {"title": "", "text": "", "summary": "", "image": "", "og_image": "", "via": _domain(url)}
- r.encoding = "utf-8"
- soup = BeautifulSoup(r.text, "lxml")
- for tag in soup.find_all(["script", "style", "nav", "footer", "aside", "form", "noscript", "iframe", ".ads", ".ad", ".banner-ads", ".fb-comments", ".fb-root", ".social-share", ".related-news", ".breadcrumb"]):
- tag.decompose()
- title = ""
- ogt = soup.find("meta", property="og:title")
- if ogt:
- title = ogt.get("content", "")
- h1 = soup.find("h1")
- if not title and h1:
- title = h1.get_text(strip=True)
- if not title:
- t = soup.find("title")
- if t:
- title = t.get_text(strip=True)
- og_image = ""
- ogi = soup.find("meta", property="og:image")
- if ogi:
- og_image = ogi.get("content", "")
- if og_image.startswith("//"):
- og_image = "https:" + og_image
- summary = ""
- ogd = soup.find("meta", property="og:description") or soup.find("meta", attrs={"name": "description"})
- if ogd:
- summary = ogd.get("content", "")[:500]
- body_text = []
- for sel in ["article", ".singular-content", ".detail-content", ".fck_detail", ".content-detail", ".knc-content", "main", ".cms-body", ".article__body", ".post-content", ".entry-content"]:
- el = soup.select_one(sel)
- if el and len(el.find_all("p")) >= 2:
- for p in el.find_all("p"):
- t = _clean_text(p.get_text(strip=True))
- if t and len(t) > 30:
- body_text.append(t)
- break
- if not body_text and soup.body:
- for p in soup.body.find_all("p"):
- t = _clean_text(p.get_text(strip=True))
- if t and len(t) > 30:
- body_text.append(t)
- text = "\n".join(body_text)
- return {"title": _clean_text(title), "text": text, "summary": _clean_text(summary), "image": og_image, "og_image": og_image, "via": _domain(url), "url": url}
+ os.makedirs(SHORTS_DIR, exist_ok=True)
+ out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
+ if os.path.exists(out_mp4):
+ return "/api/ai/short-file/" + post_id
+
+ work = os.path.join(SHORTS_DIR, _safe_name(post_id))
+ os.makedirs(work, exist_ok=True)
+ img_path = os.path.join(work, "image.jpg")
+ frame_path = os.path.join(work, "frame.jpg")
+ audio_path = os.path.join(work, "voice.mp3")
+
+ _download_image(post.get("img"), post.get("title", "AI news"), img_path)
+ _make_short_frame(post, img_path, frame_path)
+ script = _short_script(post)
+
+ if voice_id is None:
+ voice_id = _detect_voice_for_topic(post.get("title", ""), post.get("text", ""))
+ if speed is None:
+ speed = TTS_DEFAULT_SPEED
+
+ try:
+ await _generate_tts_edge(script, voice_id, speed, audio_path)
+ except Exception as e:
+ print(f"[TTS edge-tts error] {e}, falling back to gTTS")
+ if gTTS:
+ _generate_tts_gtts(script, audio_path)
+ else:
+ return ""
+
+ cmd = ["ffmpeg", "-y", "-loop", "1", "-i", frame_path, "-i", audio_path,
+ "-shortest", "-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
+ "-c:a", "aac", "-b:a", "128k", "-vf", "scale=1080:1920", out_mp4]
+ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180)
+ return "/api/ai/short-file/" + post_id
except Exception as e:
- return {"title": "", "text": "", "summary": "", "image": "", "og_image": "", "via": _domain(url)}
+ print(f"[short video error] {e}")
+ return ""
-def make_post(title: str, text: str, img: str, url: str, kind: str = "auto", sources: list = None) -> dict:
- """Create a wall post dict."""
- import random as _r2
- now = int(time.time() * 1000)
+# ===== MAKE POST =====
+def make_post(title, text, image, source_url, kind, sources=None, images=None):
return {
- "id": str(now) + str(_r2.randint(100, 999)),
- "title": (title or "Bài viết")[:200],
- "text": (text or "")[:5000],
- "img": img or "",
- "url": url or "",
- "kind": kind or "auto",
- "sources": sources or [],
- "created": now,
- "created_str": datetime.now(_VN_TZ).strftime("%H:%M %d/%m/%Y"),
+ "id": str(int(time.time() * 1000)) + str(random.randint(100, 999)),
+ "title": title, "text": text, "img": image, "url": source_url,
+ "kind": kind, "sources": sources or [], "video": "",
+ "images": images or [], "ts": int(time.time())
}
+
+
+# ===== SHARED PROMPT BUILDER — STRICT REWRITE =====
+def _build_rewrite_prompt(title: str, raw: str, images: List[str] = None) -> str:
+ image_info = ""
+ if images:
+ num = len(images)
+ if num == 1:
+ image_info = "\n\nBài viết có 1 ảnh minh họa. Hãy tham khảo ảnh để hiểu ngữ cảnh (nếu phù hợp)."
+ else:
+ image_info = f"\n\nBài viết có {num} ảnh minh họa. Hãy tham khảo tất cả ảnh để hiểu ngữ cảnh và bổ sung thông tin cho bài viết (nếu phù hợp)."
+
+ return f"""Đọc bài viết dưới đây và viết một bài TÓM TẮT NGẮN trên Tường AI.
+
+⚠️ QUY TẮC CỨNG — VI PHẠM = BỎ QUA:
+1. KHÔNG sao chép nguyên văn bất kỳ câu nào từ bài gốc. Phải diễn đạt LẠI hoàn toàn bằng ngôn ngữ riêng.
+2. KHÔNG lặp lại bất kỳ thông tin nào. Mỗi ý chỉ xuất hiện ĐÚNG 1 LẦN.
+3. Nếu 2 câu diễn đạt cùng 1 ý → chỉ giữ 1 câu, bỏ cây còn lại.
+4. KHÔNG dùng Markdown (##, **, ---, *).
+5. KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu", "Nhiệm vụ", "Vai trò", "Đây là bài tóm tắt", "Sau đây là", "Dựa trên".
+6. Viết thành ĐOẠN VĂN THUẦN, mạch lạc, dễ đọc. Không dùng bullet points.
+7. Giữ sự thật, KHÔNG bịa thông tin.
+8. Tối đa 200 từ. Ngắn gọn, đủ ý.
+9. Mỗi câu phải trọn vẹn, không bị ngắt giữa chừng.{image_info}
+
+Tiêu đề gốc: {title}
+
+Nội dung gốc:
+{raw[:14000]}
+
+Chỉ viết phần tóm tắt, không thêm gì khác:"""
+
+
+def _build_topic_prompt(topic: str, ctx: str) -> str:
+ return f"""Viết bài TÓM TẮT NGẮN GỌN về chủ đề: "{topic}".
+
+⚠️ QUY TẮC CỨNG:
+1. KHÔNG sao chép nguyên văn từ nguồn. Phải diễn đạt LẠI hoàn toàn.
+2. KHÔNG lặp lại bất kỳ nội dung nào. Mỗi thông tin chỉ xuất hiện ĐÚNG 1 LẦN.
+3. Nếu 2 câu nói cùng 1 ý → chỉ giữ 1 câu.
+4. KHÔNG dùng Markdown (##, **, ---, *).
+5. KHÔNG viết "Dưới đây là", "Tôi sẽ", "Theo yêu cầu", "Nhiệm vụ", "Vai trò".
+6. Viết thành ĐOẠN VĂN THUẦN, mạch lạc. Không dùng bullet points.
+7. Giữ sự thật, KHÔNG bịa.
+8. Tối đa 200 từ. Ngắn gọn, đủ ý.
+9. Mỗi câu phải trọn vẹn.
+
+Nguồn thực tế:
+{ctx[:12000]}
+
+Chỉ viết phần tóm tắt, không thêm gì khác:"""
+
+
+# ===== WRITE ENDPOINTS =====
+@app.post("/api/rewrite_share")
+async def api_rewrite_share(request: Request):
+ body = await request.json()
+ url = _clean_text(body.get("url", ""))
+ if not url.startswith("http"):
+ return JSONResponse({"error": "missing url"}, status_code=400)
+ try:
+ data = scrape_any_url(url)
+ except Exception as e:
+ return JSONResponse({"error": "Không đọc được bài viết: " + str(e)[:180]}, status_code=422)
+ raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
+ if len(raw) < 60:
+ return JSONResponse({"error": "Bài viết quá ngắn để tóm tắt"}, status_code=422)
+
+ images = data.get("images", [])
+ prompt = _build_rewrite_prompt(data.get("title", ""), raw, images)
+ text = await qwen_generate(prompt, image_urls=images if images else None, max_tokens=700)
+ if not text:
+ return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
+ text = _clean_ai_output(text)
+ post = make_post(data.get("title") or "Bài viết", text,
+ images[0] if images else data.get("image", ""),
+ url, "rewrite", images=images)
+
+ # Auto-generate short video
+ video_url = await _generate_short_video(post, post["id"])
+ if video_url:
+ post["video"] = video_url
+
+ posts = _load_wall()
+ posts.insert(0, post)
+ _save_wall(posts)
+ return JSONResponse({"post": post})
+
+
+@app.post("/api/url_wall")
+async def api_url_wall(request: Request):
+ body = await request.json()
+ url = _clean_text(body.get("url", ""))
+ if not url.startswith("http"):
+ return JSONResponse({"error": "missing url"}, status_code=400)
+ try:
+ data = scrape_any_url(url)
+ except Exception as e:
+ return JSONResponse({"error": "Không scrape được URL: " + str(e)[:180]}, status_code=422)
+ raw = (data.get("summary", "") + "\n" + data.get("text", "")).strip()
+ if len(raw) < 60:
+ return JSONResponse({"error": "URL không có đủ nội dung"}, status_code=422)
+
+ images = data.get("images", [])
+ prompt = _build_rewrite_prompt(data.get("title", ""), raw, images)
+ text = await qwen_generate(prompt, image_urls=images if images else None, max_tokens=700)
+ if not text:
+ return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
+ text = _clean_ai_output(text)
+ post = make_post(data.get("title") or "Bài viết", text,
+ images[0] if images else data.get("image", ""),
+ url, "url", images=images)
+
+ video_url = await _generate_short_video(post, post["id"])
+ if video_url:
+ post["video"] = video_url
+
+ posts = _load_wall()
+ posts.insert(0, post)
+ _save_wall(posts)
+ return JSONResponse({"post": post})
+
+
+@app.post("/api/topic_post")
+async def api_topic_post(request: Request):
+ body = await request.json()
+ topic = _clean_text(body.get("topic", ""))
+ if not topic:
+ return JSONResponse({"error": "missing topic"}, status_code=400)
+
+ ctx = _web_context(topic)
+ if not ctx:
+ return JSONResponse({"error": "Không lấy được dữ liệu cho chủ đề này"}, status_code=422)
+
+ image = pollinations_image_url(topic)
+ prompt = _build_topic_prompt(topic, ctx)
+ text = await qwen_generate(prompt, image_url=image, max_tokens=700)
+ if not text:
+ return JSONResponse({"error": "Qwen2.5-VL chưa sẵn sàng: " + LAST_QWEN_ERROR}, status_code=503)
+ text = _clean_ai_output(text)
+ post = make_post(topic, text, image, "", "topic")
+
+ video_url = await _generate_short_video(post, post["id"])
+ if video_url:
+ post["video"] = video_url
+
+ posts = _load_wall()
+ posts.insert(0, post)
+ _save_wall(posts)
+ return JSONResponse({"post": post})
+
+
+# ===== WALL ENDPOINTS =====
+@app.get("/api/ai_wall")
+def api_ai_wall():
+ return JSONResponse({"posts": _load_wall()[:80]})
+
+@app.get("/api/wall")
+def api_wall():
+ return JSONResponse({"posts": _load_wall()[:80]})
+
+
+# ===== SHORT VIDEO ENDPOINT =====
+@app.post("/api/ai/short/{post_id}")
+async def api_ai_short(post_id: str, voice: str = Query(default=None), speed: float = Query(default=None)):
+ posts = _load_wall()
+ post = next((p for p in posts if str(p.get("id")) == str(post_id)), None)
+ if not post:
+ return JSONResponse({"error": "post not found"}, status_code=404)
+
+ os.makedirs(SHORTS_DIR, exist_ok=True)
+ out_mp4 = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
+
+ if os.path.exists(out_mp4) and voice is None and speed is None:
+ video_url = "/api/ai/short-file/" + post_id
+ for i, p in enumerate(posts):
+ if str(p.get("id")) == str(post_id):
+ posts[i]["video"] = video_url
+ break
+ _save_wall(posts)
+ return JSONResponse({"video": video_url})
+
+ if voice is not None and voice not in TTS_VOICES:
+ return JSONResponse({"error": f"voice không hợp lệ. Chọn: {list(TTS_VOICES.keys())}"}, status_code=400)
+
+ video_url = await _generate_short_video(post, post_id, voice_id=voice, speed=speed)
+ if video_url:
+ for i, p in enumerate(posts):
+ if str(p.get("id")) == str(post_id):
+ posts[i]["video"] = video_url
+ break
+ _save_wall(posts)
+ return JSONResponse({"video": video_url})
+ return JSONResponse({"error": "Không tạo được shorts"}, status_code=500)
+
+
+@app.get("/api/ai/short-file/{post_id}")
+def api_ai_short_file(post_id: str):
+ path = os.path.join(SHORTS_DIR, _safe_name(post_id) + ".mp4")
+ if not os.path.exists(path):
+ return JSONResponse({"error": "not found"}, status_code=404)
+ return FileResponse(path, media_type="video/mp4", filename=f"vnews-ai-{post_id}.mp4")
+
+
+@app.get("/api/ai/status")
+def api_ai_status():
+ return JSONResponse({
+ "has_token": bool(_hf_token()),
+ "client_imported": AsyncInferenceClient is not None,
+ "model": QWEN_VL_MODEL,
+ "last_error": LAST_QWEN_ERROR,
+ "tts_ready": gTTS is not None or edge_tts is not None,
+ "tts_engine": "edge-tts" if edge_tts else ("gtts" if gTTS else "none"),
+ "tts_voices": {k: v["flag"] + " " + v["name"] for k, v in TTS_VOICES.items()},
+ "tts_voice_count": len(TTS_VOICES),
+ "tts_default_speed": TTS_DEFAULT_SPEED,
+ })
+
+
+@app.get("/api/ai/voices")
+def api_ai_voices():
+ voices_out = {}
+ for k, v in TTS_VOICES.items():
+ voices_out[k] = {
+ "name": v["name"],
+ "gender": v["gender"],
+ "country": v["country"],
+ "lang": v["lang"],
+ "flag": v["flag"],
+ "label": f"{v['flag']} {v['name']} ({v['gender']})",
+ }
+ return JSONResponse({
+ "voices": voices_out,
+ "default_voice": TTS_DEFAULT_VOICE,
+ "default_speed": TTS_DEFAULT_SPEED,
+ "topic_voice_map": TOPIC_VOICE_MAP,
+ })
diff --git a/ai_fix2.py b/ai_fix2.py
index 895be1d7505ed5fedaec5b8f023a52434101989f..b5a201354fe9e23c2294108b52d0ac2e3b06ca8d 100644
--- a/ai_fix2.py
+++ b/ai_fix2.py
@@ -321,7 +321,7 @@ async def ai_short_full(post_id: str, request: Request):
subprocess.run(['ffmpeg','-y','-i',audio,'-filter:a',f'atempo={speed}','-vn',audio_fast], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=220)
duration = 45.0
try:
- pr = subprocess.run(['ffprobe','-v','error','-show_entries','format=duration','-of','default=noprint_wrappers=1:no_key=1',audio_fast], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20)
+ pr = subprocess.run(['ffprobe','-v','error','-show_entries','format=duration','-of','default=noprint_wrappers=1:nokey=1',audio_fast], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20)
duration = float((pr.stdout or b'45').decode().strip() or 45)
except Exception:
pass
diff --git a/ai_patch.py b/ai_patch.py
index 41aeba3d810744429c14e473c32c3a9fb8e3a605..54e371e6a5d5ede04884ed0b4742ef289ef250ad 100644
--- a/ai_patch.py
+++ b/ai_patch.py
@@ -6,7 +6,6 @@ import json
import html as html_lib
import subprocess
import requests
-import hashlib
import ai_ext as base
from ai_ext import app
from fastapi import Request
@@ -42,17 +41,17 @@ def _similar(a, b):
return len(ta & tb) / max(1, min(len(ta), len(tb))) >= 0.72
-def _dedupe_units(units, max_units=25):
- """Deduplicate units - only skip exact matches to ensure all bullet points are read."""
+def _dedupe_units(units, max_units=7):
out, seen = [], set()
for u in units:
u = _clean(re.sub(r"^[-•*\d\.\)\s]+", "", u))
if len(u) < 18:
continue
nu = _norm(u)
- # Only skip exact matches, NOT similar content (to avoid skipping valid bullet points)
if nu in seen:
continue
+ if any(_similar(u, old) for old in out):
+ continue
seen.add(nu)
out.append(u)
if len(out) >= max_units:
@@ -60,7 +59,7 @@ def _dedupe_units(units, max_units=25):
return out
-def _postprocess_ai_text(text, max_units=20):
+def _postprocess_ai_text(text, max_units=7):
text = _clean(text)
if not text:
return text
@@ -79,9 +78,10 @@ def _postprocess_ai_text(text, max_units=20):
raw_lines.append(line)
units = []
for line in raw_lines:
- # KEEP FULL bullet point - don't truncate or split into segments
- if len(line) >= 18:
- units.append(_clean(re.sub(r"^[-•*\d\.\)\s]+", "", line)))
+ if len(line) > 260:
+ units.extend(re.split(r"(?<=[\.\!\?])\s+(?=[A-ZÀ-Ỹ0-9])", line))
+ else:
+ units.append(line)
units = _dedupe_units(units, max_units=max_units)
if not units:
return text[:900]
@@ -194,6 +194,7 @@ def _topic_source_articles(topic, limit=5):
break
except Exception:
continue
+ # Fallback to headlines/snippets only if no full body works.
if not out:
for s in (sources or _direct_news_rss(topic, 6))[:limit]:
title = s.get("title") or topic
@@ -268,7 +269,7 @@ async def qwen_generate_resilient(prompt: str, image_url=None, max_tokens: int =
errors.append("missing HF_TOKEN")
base.LAST_QWEN_ERROR = " | ".join(errors[-6:]) or "Qwen unavailable; used extractive fallback"
print("[qwen resilient fallback]", base.LAST_QWEN_ERROR)
- return _fallback_summary_from_prompt(prompt, max_units=12)
+ return _fallback_summary_from_prompt(prompt, max_units=6)
if not hasattr(base, "_original_qwen_generate"):
@@ -321,30 +322,12 @@ Yêu cầu bắt buộc:
Nội dung bài:
{art['raw'][:14000]}"""
- text = await base.qwen_generate(prompt, image_url=art.get('image') or None, max_tokens=1500)
- text = _postprocess_ai_text(text, max_units=20)
+ text = await base.qwen_generate(prompt, image_url=art.get('image') or None, max_tokens=900)
+ text = _postprocess_ai_text(text, max_units=6)
src = [art['source']]
if 'Nguồn tham khảo:' not in text:
text += "\n\n" + _source_line(src)
post = base.make_post(art['title'], text, art.get('image') or base.pollinations_image_url(art['title']), art.get('url') or '', 'topic_article', sources=src)
-
- # Generate slides for this post so they persist after page reload
- try:
- page_data = _scrape_article_images(art.get('url', ''))
- if page_data and page_data.get('paragraphs'):
- key_points = _extract_key_points_for_slides(page_data['paragraphs'], max_points=12)
- if key_points:
- relevant_imgs = page_data.get('images', [])
- if not relevant_imgs and page_data.get('og_img'):
- relevant_imgs = [page_data['og_img']]
- slides = []
- for i, point in enumerate(key_points):
- img = relevant_imgs[i] if i < len(relevant_imgs) else (relevant_imgs[-1] if relevant_imgs else '')
- slides.append({'text': point, 'image': img, 'index': i + 1})
- post['slides'] = slides
- except Exception:
- pass
-
new_posts.append(post)
posts = new_posts + posts
base._save_ai_wall(posts)
@@ -365,142 +348,14 @@ async def compat_url_wall(request: Request):
if len(raw) < 120:
return JSONResponse({'error': 'URL không có đủ nội dung để tóm tắt'}, status_code=422)
prompt = _make_summary_prompt(data.get('title', ''), raw, data.get('via', '') or base._domain(url))
- text = await base.qwen_generate(prompt, image_url=data.get('image') or None, max_tokens=1500)
- text = _postprocess_ai_text(text, max_units=20)
+ text = await base.qwen_generate(prompt, image_url=data.get('image') or None, max_tokens=850)
+ text = _postprocess_ai_text(text, max_units=6)
src = [{'title': data.get('title'), 'url': url, 'excerpt': raw[:500], 'via': data.get('via') or base._domain(url)}]
if 'Nguồn tham khảo:' not in text:
text += "\n\n" + _source_line(src)
post = base.make_post(data.get('title') or 'Bài viết', text, data.get('image') or '', url, 'url', sources=src)
-
- # Generate slides so they persist after page reload
- slides = []
- try:
- page_data = _scrape_article_images(url)
- if page_data and page_data.get('paragraphs'):
- key_points = _extract_key_points_for_slides(page_data['paragraphs'], max_points=12)
- if key_points:
- relevant_imgs = page_data.get('images', [])
- if not relevant_imgs and page_data.get('og_img'):
- relevant_imgs = [page_data['og_img']]
- for i, point in enumerate(key_points):
- img = relevant_imgs[i] if i < len(relevant_imgs) else (relevant_imgs[-1] if relevant_imgs else '')
- slides.append({'text': point, 'image': img, 'index': i + 1})
- except Exception:
- pass
- post['slides'] = slides
-
posts = base._load_ai_wall(); posts.insert(0, post); base._save_ai_wall(posts)
- return JSONResponse({'post': post, 'slides': slides})
-
-
-def _is_relevant_image(img_url, title, text):
- """Check if an image is relevant to the article content."""
- if not img_url:
- return False
- skip_patterns = ['pixel', 'analytics', 'tracking', '1x1.gif', 'spacer.gif',
- 'logo', 'icon', 'avatar', 'emoji', 'smiley', 'sprite',
- 'advertisement', 'ad-banner', 'sponsored', 'banner-ads']
- img_lower = img_url.lower()
- for p in skip_patterns:
- if p in img_lower:
- return False
- if not any(img_lower.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.webp', '.gif']):
- return False
- return True
-
-
-def _filter_relevant_images(images, title, text, max_images=8):
- """Filter and rank images by relevance to article content."""
- if not images:
- return []
- seen = set()
- relevant = []
- for img in images:
- if img in seen:
- continue
- seen.add(img)
- if _is_relevant_image(img, title, text):
- relevant.append(img)
- return relevant[:max_images]
-
-
-def _extract_key_points_for_slides(paragraphs, max_points=12):
- """Extract key points from paragraphs for slides - extracts ALL sentences, not just first one."""
- points = []
- for p in paragraphs:
- if len(points) >= max_points:
- break
- p = _clean(p)
- if not p:
- continue
- # Split paragraph into sentences using Vietnamese + English punctuation - GET ALL SENTENCES
- sentences = re.split(r'(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])', p)
- sentences = [s.strip() for s in sentences if s.strip()]
-
- for sentence in sentences:
- if len(points) >= max_points:
- break
- sentence = _clean(sentence)
- if len(sentence) < 30:
- continue
- if any(sentence[:60] in existing for existing in points):
- continue
- if not sentence.endswith(('.', '!', '?')):
- sentence = sentence + '.'
- points.append(sentence)
- return points
-
-
-def _scrape_article_images(url):
- """Scrape article page and return only relevant images."""
- try:
- headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
- "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"}
- r = requests.get(url, headers=headers, timeout=15, allow_redirects=True)
- r.encoding = 'utf-8'
- soup = BeautifulSoup(r.text, 'lxml')
- for tag in soup.find_all(['script', 'style', 'nav', 'footer', 'aside', 'form']):
- tag.decompose()
- h1 = soup.find('h1')
- ogt = soup.find('meta', property='og:title')
- title = (h1.get_text(strip=True) if h1 else '') or (ogt.get('content', '') if ogt else '')
- ogi = soup.find('meta', property='og:image')
- og_img = ogi.get('content', '') if ogi else ''
- if og_img and og_img.startswith('//'):
- og_img = 'https:' + og_img
- block = None
- for sel in ['article', '.singular-content', '.detail-content', '.fck_detail', '.content-detail', '.knc-content', 'main', '.cms-body', '.article__body']:
- el = soup.select_one(sel)
- if el and len(el.find_all('p')) >= 2:
- block = el
- break
- if not block:
- block = soup.body or soup
- paragraphs = []
- all_images = []
- seen_imgs = set()
- if og_img and og_img not in seen_imgs:
- all_images.append(og_img)
- seen_imgs.add(og_img)
- for el in block.find_all(['p', 'h2', 'h3', 'figure', 'img'], recursive=True):
- if el.name == 'p':
- t = _clean(el.get_text(strip=True))
- if t and len(t) > 40:
- paragraphs.append(t)
- elif el.name in ('figure', 'img'):
- im = el if el.name == 'img' else el.find('img')
- if im:
- src = im.get('data-src') or im.get('src') or im.get('data-original') or ''
- if src and 'base64' not in src:
- if src.startswith('//'):
- src = 'https:' + src
- if src not in seen_imgs:
- all_images.append(src)
- seen_imgs.add(src)
- relevant_images = _filter_relevant_images(all_images, title, ' '.join(paragraphs[:5]))
- return {'title': _clean(title), 'paragraphs': paragraphs, 'images': relevant_images, 'og_img': og_img}
- except Exception:
- return None
+ return JSONResponse({'post': post})
@app.post('/api/rewrite_share')
@@ -517,56 +372,40 @@ async def compat_rewrite_share(request: Request):
if len(raw) < 120:
return JSONResponse({'error': 'Bài viết không đủ nội dung để tóm tắt'}, status_code=422)
prompt = _make_summary_prompt(data.get('title', ''), raw, data.get('via', '') or base._domain(url))
- text = await base.qwen_generate(prompt, image_url=data.get('image') or None, max_tokens=1500)
- text = _postprocess_ai_text(text, max_units=20)
+ text = await base.qwen_generate(prompt, image_url=data.get('image') or None, max_tokens=850)
+ text = _postprocess_ai_text(text, max_units=6)
src = [{'title': data.get('title'), 'url': url, 'excerpt': raw[:500], 'via': data.get('via') or base._domain(url)}]
if 'Nguồn tham khảo:' not in text:
text += "\n\n" + _source_line(src)
post = base.make_post(data.get('title') or 'Bài viết', text, data.get('image') or '', url, 'summary', sources=src)
-
- # Generate slides with relevant images only
- slides = []
- page_data = _scrape_article_images(url)
- if page_data and page_data.get('paragraphs'):
- key_points = _extract_key_points_for_slides(page_data['paragraphs'], max_points=12)
- if key_points:
- relevant_imgs = page_data.get('images', [])
- if not relevant_imgs and page_data.get('og_img'):
- relevant_imgs = [page_data['og_img']]
- for i, point in enumerate(key_points):
- img = relevant_imgs[i] if i < len(relevant_imgs) else (relevant_imgs[-1] if relevant_imgs else '')
- slides.append({'text': point, 'image': img, 'index': i + 1})
-
- # FIX: Save slides into post so they persist after page reload
- post['slides'] = slides
posts = base._load_ai_wall(); posts.insert(0, post); base._save_ai_wall(posts)
-
- return JSONResponse({'post': post, 'slides': slides})
+ return JSONResponse({'post': post})
def _emotion_script(text, emotion):
- """Prepend emotion-appropriate prefix to text based on emotion type.
-
- NOTE: Prefix is NOT added to avoid cluttering Short AI speech.
- The emotion is still used for voice selection but content is read cleanly.
- """
text = _clean(text)
- # REMOVED: No prefix added to keep content clean and natural
+ if emotion == 'urgent':
+ return 'Tin nhanh. ' + text
+ if emotion == 'warm':
+ return 'Câu chuyện đáng chú ý. ' + text
+ if emotion == 'serious':
+ return 'Bản tin nghiêm túc. ' + text
+ if emotion == 'energetic':
+ return 'Cập nhật nổi bật. ' + text
return text
def _tts_script_smart(post, emotion):
- raw = base._short_script(post) if hasattr(base, '_short_script') else _clean(post.get('text', '') or post.get('title', ''))
+ raw = base._short_script(post)
raw = re.sub(r"^[•\-\*]\s*", "", raw, flags=re.M)
raw = re.sub(r"\s*\n\s*", ". ", raw)
raw = re.sub(r"([\.\!\?])\s*", r"\1\n", raw)
raw = re.sub(r"\n{2,}", "\n", raw).strip()
- # REMOVED: _emotion_script call - read content cleanly without prefix
- # INCREASED to 3000 to read full content of all bullet points
- if len(raw) > 3000:
- raw = raw[:3000]
+ raw = _emotion_script(raw, emotion)
+ if len(raw) > 1000:
+ raw = raw[:1000]
cut = max(raw.rfind("."), raw.rfind("!"), raw.rfind("?"))
- if cut > 700:
+ if cut > 350:
raw = raw[:cut + 1]
return raw
@@ -681,7 +520,8 @@ def _make_short_frame_full(post, img_path, out_path):
-def _summary_segments_from_post(post, max_segments=25):
+def _summary_segments_from_post(post, max_segments=7):
+ """Only use current post summary. Each bullet/paragraph becomes one short scene."""
raw = _clean(post.get('text') or post.get('title') or '')
raw = re.sub(r'^Bản tin AI viết lại:\s*', '', raw, flags=re.I)
raw = re.sub(r'Nguồn tham khảo:.*$', '', raw, flags=re.I|re.S).strip()
@@ -692,7 +532,7 @@ def _summary_segments_from_post(post, max_segments=25):
low=ln.lower()
if low.startswith(('điểm chính','tiêu đề','sapo','nguồn tham khảo')): continue
if len(ln)>=18: lines.append(ln)
- if len(lines)<3:
+ if len(lines)<2:
lines=[]
for s in re.split(r'(?<=[\.\!\?])\s+', raw):
s=_clean(s)
@@ -702,12 +542,14 @@ def _summary_segments_from_post(post, max_segments=25):
def _make_scene_frame(post, segment, idx, total, img_path, out_path, emotion='neutral'):
+ """Create one 9:16 frame for one summary segment. No subtitle layer."""
if Image is None:
return _make_short_frame_full(post, img_path, out_path)
W,H=1080,1920
bg=Image.new('RGB',(W,H),(10,10,10))
try:
im=Image.open(img_path).convert('RGB')
+ # background cover
ratio=im.width/max(1,im.height); target=W/H
if ratio>target:
nh=H; nw=int(nh*ratio)
@@ -717,6 +559,7 @@ def _make_scene_frame(post, segment, idx, total, img_path, out_path, emotion='ne
cover=cover.crop((left,top,left+W,top+H))
bg.paste(cover,(0,0))
bg=Image.blend(bg, Image.new('RGB',(W,H),(0,0,0)), 0.50)
+ # top hero image
hero_h=720; target=W/hero_h
if ratio>target:
nh=hero_h; nw=int(nh*ratio)
@@ -736,6 +579,7 @@ def _make_scene_frame(post, segment, idx, total, img_path, out_path, emotion='ne
except Exception:
font_brand=font_small=font_seg=font_title=None
draw.rectangle((0,680,W,H), fill=(12,12,12))
+ # progress bars
dot_x=48; dot_y=742
for i in range(total):
fill=(92,184,122) if i==idx else (70,70,70)
@@ -744,8 +588,7 @@ def _make_scene_frame(post, segment, idx, total, img_path, out_path, emotion='ne
draw.rounded_rectangle((48,834,260,880), radius=20, fill=(28,70,45))
draw.text((66,842),f'Đoạn {idx+1}/{total}',fill=(235,235,235),font=font_small)
y=940; maxw=W-96
- # INCREASED from 12 to 18 for full content display - each key point can span multiple lines
- for ln in _wrap_text_px(draw, segment, font_seg, maxw, 18):
+ for ln in _wrap_text_px(draw, segment, font_seg, maxw, 8):
draw.text((48,y),ln,fill=(255,255,255),font=font_seg)
y+=74
if y>1500: break
@@ -757,24 +600,24 @@ def _make_scene_frame(post, segment, idx, total, img_path, out_path, emotion='ne
bg.save(out_path, quality=92)
-def _estimate_audio_duration(path, fallback=15.0):
- """Estimate audio duration with 15s minimum per segment for complete bullet reading."""
+def _estimate_audio_duration(path, fallback=4.0):
try:
- pr=subprocess.run(['ffprobe','-v','error','-show_entries','format=duration','-of','default=noprint_wrappers=1:no_key=1',path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20)
- return max(12.0, float((pr.stdout or b'').decode().strip() or fallback))
+ pr=subprocess.run(['ffprobe','-v','error','-show_entries','format=duration','-of','default=noprint_wrappers=1:nokey=1',path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20)
+ return max(1.5, float((pr.stdout or b'').decode().strip() or fallback))
except Exception:
return fallback
@app.post('/api/ai/short/{post_id}')
async def patched_ai_short(post_id: str, request: Request):
+ """Create 9:16 short from separate timed summary segments; no subtitles."""
try:
body = await request.json()
except Exception:
body = {}
voice = str(body.get('voice', 'nu')).strip().lower()
emotion = str(body.get('emotion', 'neutral')).strip().lower()
- speed = float(body.get('speed', 1.0) or 1.0)
+ speed = float(body.get('speed', 1.2) or 1.2)
speed = max(0.85, min(1.35, speed))
posts = base._load_ai_wall()
@@ -782,7 +625,7 @@ async def patched_ai_short(post_id: str, request: Request):
if not post:
return JSONResponse({'error': 'post not found'}, status_code=404)
- segments = _summary_segments_from_post(post, max_segments=25)
+ segments = _summary_segments_from_post(post, max_segments=7)
seg_hash = hashlib.md5(('|'.join(segments)+voice+emotion+str(speed)).encode('utf-8')).hexdigest()[:8]
os.makedirs(base.SHORTS_DIR, exist_ok=True)
suffix = f"_{voice}_{emotion}_{str(speed).replace('.', 'p')}_{seg_hash}_scenes_nosub"
@@ -805,63 +648,11 @@ async def patched_ai_short(post_id: str, request: Request):
try:
base._download_image(post.get('img'), post.get('title', 'AI news'), img)
edge_voice = {
- # Vietnamese
- 'vi-vn-hoaimyneural': 'vi-VN-HoaiMyNeural',
- 'vi-vn-namminhneural': 'vi-VN-NamMinhNeural',
- 'hoaimy': 'vi-VN-HoaiMyNeural',
- 'namminh': 'vi-VN-NamMinhNeural',
'nam': 'vi-VN-NamMinhNeural',
'male': 'vi-VN-NamMinhNeural',
'nu': 'vi-VN-HoaiMyNeural',
'female': 'vi-VN-HoaiMyNeural',
'mien-nam': 'vi-VN-HoaiMyNeural',
- # English - Multilingual
- 'en-us-andrewmultilingualneural': 'en-US-AndrewMultilingualNeural',
- 'en-au-williammultilingualneural': 'en-AU-WilliamMultilingualNeural',
- 'andrew': 'en-US-AndrewMultilingualNeural',
- 'en_andrew': 'en-US-AndrewMultilingualNeural',
- 'jenny': 'en-US-AndrewMultilingualNeural',
- 'en_jenny': 'en-US-AndrewMultilingualNeural',
- # Portuguese - Multilingual (ONLY Thalita)
- 'pt-br-thalitamultilingualneural': 'pt-BR-ThalitaMultilingualNeural',
- 'thalita': 'pt-BR-ThalitaMultilingualNeural',
- 'pt_thalita': 'pt-BR-ThalitaMultilingualNeural',
- 'pt_br_thalita': 'pt-BR-ThalitaMultilingualNeural',
- 'pt': 'pt-BR-ThalitaMultilingualNeural',
- 'pt_francisco': 'pt-BR-ThalitaMultilingualNeural',
- # French - Multilingual
- 'fr-fr-viviennemultilingualneural': 'fr-FR-VivienneMultilingualNeural',
- 'fr-fr-remymultilingualneural': 'fr-FR-RemyMultilingualNeural',
- 'denise': 'fr-FR-VivienneMultilingualNeural',
- 'fr': 'fr-FR-VivienneMultilingualNeural',
- 'fr_denise': 'fr-FR-VivienneMultilingualNeural',
- # German - Multilingual
- 'de-de-seraphinamultilingualneural': 'de-DE-SeraphinaMultilingualNeural',
- 'de-de-florianmultilingualneural': 'de-DE-FlorianMultilingualNeural',
- 'katja': 'de-DE-SeraphinaMultilingualNeural',
- 'de': 'de-DE-SeraphinaMultilingualNeural',
- 'de_katja': 'de-DE-SeraphinaMultilingualNeural',
- # Korean - Multilingual (Hyunsu, NOT SunHee)
- 'ko-kr-hyusumultilingualneural': 'ko-KR-HyunsuMultilingualNeural',
- 'ko-kr-hyunsuneural': 'ko-KR-HyunsuMultilingualNeural',
- 'sunhee': 'ko-KR-HyunsuMultilingualNeural',
- 'ko': 'ko-KR-HyunsuMultilingualNeural',
- 'ko_sunhee': 'ko-KR-HyunsuMultilingualNeural',
- # Italian - Multilingual
- 'it-it-giuseppemultilingualneural': 'it-IT-GiuseppeMultilingualNeural',
- # Spanish (keep for backward compat)
- 'ela': 'en-US-AndrewMultilingualNeural',
- 'es_ela': 'en-US-AndrewMultilingualNeural',
- 'es': 'en-US-AndrewMultilingualNeural',
- 'es_carlos': 'en-US-AndrewMultilingualNeural',
- # Japanese (keep for backward compat)
- 'nanami': 'en-US-AndrewMultilingualNeural',
- 'ja': 'en-US-AndrewMultilingualNeural',
- 'ja_nanami': 'en-US-AndrewMultilingualNeural',
- # Chinese (keep for backward compat)
- 'xiaochen': 'en-US-AndrewMultilingualNeural',
- 'zh': 'en-US-AndrewMultilingualNeural',
- 'zh_xiaochen': 'en-US-AndrewMultilingualNeural',
}.get(voice, 'vi-VN-HoaiMyNeural')
part_files=[]
for idx, seg in enumerate(segments):
@@ -874,13 +665,14 @@ async def patched_ai_short(post_id: str, request: Request):
try:
subprocess.run(['python','-m','edge_tts','--voice',edge_voice,'--text',spoken,'--write-media',aud], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120)
except Exception:
- tld='com.vn' if voice in ('nu','female','mien-nam','hoaimy') else 'com'
+ tld='com.vn' if voice in ('nu','female','mien-nam') else 'com'
try:
base.gTTS(spoken, lang='vi', tld=tld, slow=False).save(aud)
except TypeError:
base.gTTS(spoken, lang='vi', slow=False).save(aud)
subprocess.run(['ffmpeg','-y','-i',aud,'-filter:a',f'atempo={speed}','-vn',aud_fast], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=90)
- dur=_estimate_audio_duration(aud_fast, fallback=15.0)+0.35
+ dur=_estimate_audio_duration(aud_fast, fallback=4.0)+0.35
+ # No subtitles: scene text is part of frame only.
subprocess.run(['ffmpeg','-y','-loop','1','-t',str(dur),'-i',frame,'-i',aud_fast,'-shortest','-c:v','libx264','-tune','stillimage','-pix_fmt','yuv420p','-c:a','aac','-b:a','128k',part], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=150)
part_files.append(part)
concat=os.path.join(work,'concat.txt')
@@ -915,3 +707,33 @@ def api_ai_shorts():
app.router.routes = [r for r in app.router.routes if not (getattr(r, 'path', None) == '/' and 'GET' in getattr(r, 'methods', set()))]
+
+PATCH_INJECT = r'''
+
+🎬 Đang tạo Short AI
Đang chuẩn bị...
', PATCH_INJECT+'\n'))
diff --git a/ai_runtime_final.py b/ai_runtime_final.py
index 8dce363f4414b76f00d41b1e4cccc2c2c3809d78..1e23ae3e8836bc9090b91a1bd1899314fdf291b1 100644
--- a/ai_runtime_final.py
+++ b/ai_runtime_final.py
@@ -297,7 +297,7 @@ async def final_short(post_id:str,request:Request):
clips.append(clip)
lf=os.path.join(work,'list.txt')
with open(lf,'w',encoding='utf-8') as f:
- for c in clips:f.write("file '"+c.replace("","'\\''"))+"'\n")
+ for c in clips:f.write("file '"+c.replace("'","'\\''")+"'\n")
subprocess.run(['ffmpeg','-y','-f','concat','-safe','0','-i',lf,'-c','copy',out],check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=240)
post['video']='/api/ai/short-file/'+post_id+suffix;post['short_subtitles']=False;post['short_segments']=segs;post['short_speed']=speed;base._save_ai_wall(posts)
return JSONResponse({'video':post['video'],'segments':len(segs),'subtitles':False})
@@ -309,7 +309,48 @@ def ai_wall_share(post:str=Query(default=''), short:int=Query(default=0)):
if not p:return HTMLResponse(f'')
title=p.get('title') or 'VNEWS AI';img=p.get('img') or DEFAULT_IMG
desc=(p.get('text') or '')[:220]
- return HTMLResponse(f'