bep40 commited on
Commit
3d2a783
·
verified ·
1 Parent(s): 97f13fb

revert: restore main.py to previous working version

Browse files
Files changed (1) hide show
  1. main.py +178 -307
main.py CHANGED
@@ -1,8 +1,8 @@
1
- """VNEWS - FastAPI backend with persistent JSON database + hash-based share URLs."""
2
- import hashlib, re, time, json, os, threading, unicodedata
3
  from concurrent.futures import ThreadPoolExecutor, as_completed
4
- from fastapi import FastAPI, Query, Body, Request
5
- from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
6
  from fastapi.staticfiles import StaticFiles
7
  from urllib.parse import unquote, quote
8
  import requests
@@ -13,371 +13,232 @@ HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/5
13
  BASE_BDP = "https://bongdaplus.vn"
14
  BASE_24H = "https://www.24h.com.vn"
15
  SPACE_URL = "https://bep40-vnews.hf.space"
 
 
16
 
17
- # ═══ Slug/Hash helpers ═══
18
- def _remove_accents(s):
19
- # Handle đ/Đ first (NFKD doesn't decompose these)
20
- s = s.replace('đ', 'd').replace('Đ', 'D')
21
- nfkd = unicodedata.normalize('NFKD', s)
22
- return ''.join(c for c in nfkd if not unicodedata.combining(c))
23
-
24
- def make_slug(title):
25
- s = _remove_accents(title.lower())
26
- s = re.sub(r'[^a-z0-9\s-]', '', s)
27
- s = re.sub(r'[\s_]+', '-', s.strip())
28
- s = re.sub(r'-+', '-', s).strip('-')
29
- return s[:60]
30
-
31
- def make_hash(url):
32
- return hashlib.md5(url.encode()).hexdigest()[:12]
33
-
34
- # ═══ Share registry (maps hash → article info for og:image) ═══
35
- SHARE_DB = "/data/shares.json" if os.path.isdir("/data") else "./shares.json"
36
- _share_lock = threading.Lock()
37
-
38
- def _load_shares():
39
- try:
40
- with open(SHARE_DB, "r") as f: return json.load(f)
41
- except: return {}
42
-
43
- def _save_shares(db):
44
- try:
45
- os.makedirs(os.path.dirname(SHARE_DB) if os.path.dirname(SHARE_DB) else ".", exist_ok=True)
46
- with open(SHARE_DB, "w") as f: json.dump(db, f, ensure_ascii=False)
47
- except: pass
48
-
49
- def register_share(url, title, img=""):
50
- h = make_hash(url)
51
- slug = make_slug(title)
52
- with _share_lock:
53
- db = _load_shares()
54
- db[h] = {"url": url, "title": title, "img": img, "slug": slug}
55
- _save_shares(db)
56
- return slug, h
57
-
58
- def get_share(h):
59
- db = _load_shares()
60
- return db.get(h, None)
61
-
62
- # ═══ Persistent Stats Database ═══
63
- DB_PATH = "/data/vnews_db.json" if os.path.isdir("/data") else "./vnews_db.json"
64
- _db_lock = threading.Lock()
65
-
66
- def _load_db():
67
- try:
68
- with open(DB_PATH, "r") as f: return json.load(f)
69
- except: return {"views":{},"shares":{},"comments":{}}
70
-
71
- def _save_db(db):
72
- try:
73
- os.makedirs(os.path.dirname(DB_PATH) if os.path.dirname(DB_PATH) else ".", exist_ok=True)
74
- with open(DB_PATH, "w") as f: json.dump(db, f, ensure_ascii=False)
75
- except: pass
76
-
77
- def db_inc_view(aid):
78
- with _db_lock:
79
- db=_load_db(); db["views"][aid]=db["views"].get(aid,0)+1; _save_db(db); return db["views"][aid]
80
-
81
- def db_inc_share(aid):
82
- with _db_lock:
83
- db=_load_db(); db["shares"][aid]=db["shares"].get(aid,0)+1; _save_db(db); return db["shares"][aid]
84
-
85
- def db_add_comment(aid, name, text):
86
- with _db_lock:
87
- db=_load_db()
88
- if aid not in db["comments"]: db["comments"][aid]=[]
89
- db["comments"][aid].append({"n":name,"t":text,"d":time.strftime("%d/%m/%Y %H:%M")})
90
- _save_db(db); return db["comments"][aid]
91
-
92
- def db_get_stats(aid):
93
- db=_load_db()
94
- return {"views":db["views"].get(aid,0),"shares":db["shares"].get(aid,0),"comments":db["comments"].get(aid,[])}
95
-
96
- # ═══ Database API endpoints ═══
97
- @app.post("/api/view")
98
- def api_view(aid:str=Body(...,embed=True)):
99
- v=db_inc_view(aid); return JSONResponse({"views":v})
100
-
101
- @app.post("/api/share_count")
102
- def api_share_count(aid:str=Body(...,embed=True)):
103
- v=db_inc_share(aid); return JSONResponse({"shares":v})
104
-
105
- @app.post("/api/comment")
106
- def api_comment(aid:str=Body(...),name:str=Body(...),text:str=Body(...)):
107
- cmts=db_add_comment(aid,name,text); return JSONResponse({"comments":cmts})
108
-
109
- @app.get("/api/stats/{aid}")
110
- def api_stats(aid:str):
111
- return JSONResponse(db_get_stats(aid))
112
-
113
- # ═══ Share registration API ═══
114
- @app.post("/api/register_share")
115
- def api_register_share(url:str=Body(...), title:str=Body(...), img:str=Body("")):
116
- slug, h = register_share(url, title, img)
117
- aid = make_hash(url)
118
- db_inc_share(aid)
119
- return JSONResponse({"slug": slug, "hash": h, "share_url": f"{SPACE_URL}/#/{slug}/{h}", "social_url": f"{SPACE_URL}/s/{slug}/{h}"})
120
-
121
- # ═══ Server-side share page (SEO og:image for social media crawlers) ═══
122
- @app.get("/s/{slug}/{hash_id}")
123
- async def share_page(slug: str, hash_id: str):
124
- info = get_share(hash_id)
125
- if not info:
126
- return HTMLResponse(f'<!DOCTYPE html><html><head><meta charset="utf-8"><meta http-equiv="refresh" content="0;url={SPACE_URL}"></head><body></body></html>')
127
- title = info.get("title", "VNEWS")
128
- img = info.get("img", "")
129
- og_img = img if img else "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
130
- hash_url = f"{SPACE_URL}/#/{slug}/{hash_id}"
131
- # Escape for HTML attributes
132
- title_esc = title.replace('"', '&quot;').replace('<', '&lt;')
133
- return HTMLResponse(f'''<!DOCTYPE html><html><head>
134
- <meta charset="utf-8"><title>{title_esc} - VNEWS</title>
135
- <meta property="og:title" content="{title_esc}">
136
- <meta property="og:image" content="{og_img}">
137
- <meta property="og:url" content="{hash_url}">
138
- <meta property="og:type" content="article">
139
- <meta property="og:site_name" content="VNEWS - Tin Tức Việt Nam">
140
- <meta property="og:description" content="{title_esc}">
141
- <meta name="twitter:card" content="summary_large_image">
142
- <meta name="twitter:title" content="{title_esc}">
143
- <meta name="twitter:image" content="{og_img}">
144
- </head><body><script>location.href="{hash_url}";</script>
145
- <noscript><a href="{hash_url}">Xem bài viết</a></noscript></body></html>''')
146
-
147
- # ═══ Cache + scrapers ═══
148
- _cache={}; _cache_ttl=300
149
- def _cached(key,fn):
150
  now=time.time()
151
- if key in _cache and now-_cache[key]["t"]<_cache_ttl: return _cache[key]["d"]
152
- try: data=fn()
153
- except: data=_cache.get(key,{}).get("d",[])
154
- _cache[key]={"d":data,"t":now}; return data
155
 
156
- def _get(url, headers=None):
157
- h = headers or HEADERS
158
- r = requests.get(url, headers=h, timeout=15); r.encoding = "utf-8"
159
- return BeautifulSoup(r.text, "lxml")
160
 
161
  def scrape_vne(cat_url):
162
  try:
163
- soup=_get(cat_url); arts=[]
164
  for it in soup.select("article.item-news")[:15]:
165
  a=it.select_one("h2.title-news a") or it.select_one("h3.title-news a")
166
- if not a: continue
167
- t=a.get("title","") or a.get_text(strip=True); lk=a.get("href","")
168
- if not t or not lk: continue
169
- im=it.find("img"); img=(im.get("data-src") or im.get("src","")) if im else ""
170
- if img and 'blank' in img:
171
  src=it.find("source")
172
- img=src.get("srcset","").split(",")[0].strip().split(" ")[0] if src else ""
173
  desc=it.select_one("p.description")
174
  arts.append({"title":t,"link":lk,"img":img,"summary":(desc.get_text(strip=True)[:150] if desc else ""),"source":"vne"})
175
  return arts
176
- except: return []
177
 
178
  def scrape_vne_article(url):
179
  try:
180
- soup=_get(url); h1=soup.select_one("h1.title-detail"); desc=soup.select_one("p.description")
181
- og=soup.find("meta",property="og:image"); og_img=og.get("content","") if og else ""
182
- cd=soup.select_one("article.fck_detail"); body=[]
183
  if cd:
184
  for ch in cd.children:
185
- if not hasattr(ch,'name') or not ch.name: continue
186
- if ch.name=="p":
187
- t=ch.get_text(strip=True)
188
- if t: body.append({"type":"p","text":t})
189
  elif ch.name=="figure":
190
  im=ch.find("img")
191
- if im:
192
- s=im.get("data-src") or im.get("src","")
193
- cap=ch.find("figcaption")
194
- if s: body.append({"type":"img","src":s,"alt":cap.get_text(strip=True) if cap else ""})
195
- elif ch.name in ("h2","h3"):
196
- body.append({"type":"heading","text":ch.get_text(strip=True)})
197
- return {"title":h1.get_text(strip=True) if h1 else "","summary":desc.get_text(strip=True) if desc else "","og_image":og_img,"body":body,"source":"vne","url":url}
198
- except: return None
199
 
200
  def scrape_bdp(url):
201
  try:
202
- soup=_get(url); arts=[]; seen=set()
203
- for sel in ["div.news.fst","div.sld-itm.news","li.news"]:
204
  for it in soup.select(sel):
205
  tag=it.find("a",class_="title") or it.find("a",href=True)
206
- if not tag: continue
207
- t=tag.get_text(strip=True); lk=tag.get("href","")
208
- if not t or len(t)<5: continue
209
- if lk and not lk.startswith("http"): lk=BASE_BDP+lk
210
- if lk in seen: continue
211
- im=it.find("img"); img=(im.get("data-src") or im.get("src","")) if im else ""
212
- seen.add(lk); arts.append({"title":t,"link":lk,"img":img,"source":"bdp","summary":""})
213
  return arts[:15]
214
- except: return []
215
 
216
  def scrape_bdp_videos():
217
  try:
218
- soup=_get(f"{BASE_BDP}/video"); arts=[]; seen=set()
219
  for a in soup.find_all("a",href=True):
220
  href=a.get("href","")
221
- if "/video/" not in href or href in ("/video/","/video/ban-thang-dep","/video/highlight"): continue
222
- if not href.startswith("http"): href=BASE_BDP+href
223
- if href in seen: continue
224
  title=re.sub(r'^\d{2}:\d{2}','',a.get_text(strip=True)).strip()
225
- if not title or len(title)<5: continue
226
- img_tag=a.find("img") or (a.parent.find("img") if a.parent else None)
227
  img=(img_tag.get("data-src") or img_tag.get("src","")) if img_tag else ""
228
- seen.add(href); arts.append({"title":title,"link":href,"img":img,"source":"bdp"})
229
  return arts[:20]
230
- except: return []
231
 
232
  def scrape_24h_highlights():
233
  try:
234
- soup=_get(f"{BASE_24H}/video-highlight-c953.html"); arts=[]; seen=set()
235
  for a in soup.find_all("a",href=True):
236
- href=a.get("href",""); title=a.get("title","") or a.get_text(strip=True)
237
- if not title or len(title)<15 or "javascript:" in href: continue
238
- if not href.startswith("http"): href=BASE_24H+href
239
- if href in seen or "video" not in href.lower() or href.endswith("-c953.html"): continue
240
  seen.add(href)
241
- img=a.find("img"); img_src=""
242
- if img: img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
243
- if img_src and "base64" in img_src: img_src=""
244
  arts.append({"title":title,"link":href,"img":img_src,"source":"24h"})
245
  def _og(art):
246
- if art["img"]: return
247
  try:
248
- r=requests.get(art["link"],headers=HEADERS,timeout=8); r.encoding="utf-8"
249
  og=BeautifulSoup(r.text,"lxml").find("meta",property="og:image")
250
- art["img"]=og.get("content","") if og else ""
251
- except: pass
252
- need=[a for a in arts if not a["img"]][:8]
253
  if need:
254
- with ThreadPoolExecutor(4) as ex: list(ex.map(_og, need))
255
  return arts[:30]
256
- except: return []
257
 
258
  def scrape_24h_shorts():
259
  try:
260
- r=requests.get(f"{BASE_24H}/video/video-tin-tuc-cvd769.html",headers=HEADERS,timeout=15); r.encoding="utf-8"
261
- soup=BeautifulSoup(r.text,"lxml"); arts=[]; seen=set()
262
  for art in soup.find_all("article"):
263
  a=art.find("a",href=True)
264
- if not a: continue
265
  href=a.get("href","")
266
- if not href.startswith("http"): href=BASE_24H+href
267
- if href in seen: continue
268
  img_tag=art.find("img")
269
  title=(img_tag.get("alt","") if img_tag else "") or a.get("title","") or a.get_text(strip=True)
270
- if not title or len(title)<10: continue
271
  img_src=""
272
- if img_tag: img_src=img_tag.get("data-original") or img_tag.get("data-src") or img_tag.get("src","")
273
- if img_src and "base64" in img_src: img_src=""
274
- seen.add(href); arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
275
  if len(arts)<3:
276
  for a in soup.find_all("a",href=True):
277
  href=a.get("href","")
278
- if not href.endswith(".html") or "javascript:" in href or "-cvd" in href: continue
279
- if not href.startswith("http"): href=BASE_24H+href
280
- if href in seen: continue
281
- img=a.find("img") or (a.parent.find("img") if a.parent else None)
282
- if not img: continue
283
  img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
284
- if not img_src or "base64" in img_src: continue
285
  title=img.get("alt","") or a.get("title","") or a.get_text(strip=True)
286
- if not title or len(title)<8: continue
287
- seen.add(href); arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
288
  return arts[:20]
289
- except: return []
290
 
291
  def scrape_bbc_vietnamese():
292
  try:
293
  bbc_h={"User-Agent":"Mozilla/5.0","Accept-Language":"en-GB,en;q=0.9"}
294
- r=requests.get("https://www.bbc.com/vietnamese",headers=bbc_h,timeout=15); r.encoding="utf-8"
295
- soup=BeautifulSoup(r.text,"lxml"); arts=[]; seen=set()
296
  for a in soup.select("a[href*='/vietnamese/']"):
297
  href=a.get("href","")
298
- if not href or href=="/vietnamese" or href.count("/")<3: continue
299
- if not href.startswith("http"): href="https://www.bbc.com"+href
300
- if href in seen: continue
301
  title=a.get_text(strip=True)
302
- if not title or len(title)<15 or any(x in title.lower() for x in ["đăng nhập","trang chủ","bbc news"]): continue
303
- img=""
304
- container=a.parent
305
  for _ in range(3):
306
  if container:
307
  im=container.find("img")
308
- if im:
309
- img=im.get("src","") or im.get("data-src","")
310
- break
311
  container=container.parent
312
- seen.add(href); arts.append({"title":title,"link":href,"img":img if img else "","source":"bbc","summary":""})
313
- if len(arts)>=15: break
314
  return arts
315
- except: return []
316
 
317
  def scrape_bbc_article(url):
318
  try:
319
- r=requests.get(url,headers={"User-Agent":"Mozilla/5.0","Accept-Language":"en-GB"},timeout=15); r.encoding="utf-8"
320
- soup=BeautifulSoup(r.text,"lxml"); h1=soup.find("h1")
321
- og=soup.find("meta",property="og:image"); og_img=og.get("content","") if og else ""
322
  body=[]
323
  for p in soup.select("[data-component='text-block'] p, article p, main p"):
324
  t=p.get_text(strip=True)
325
- if t and len(t)>20: body.append({"type":"p","text":t})
326
- return {"title":h1.get_text(strip=True) if h1 else "","summary":"","og_image":og_img,"body":body,"source":"bbc","url":url}
327
- except: return None
 
 
 
328
 
329
  def extract_video_url(article_url):
330
  try:
331
- r=requests.get(article_url,headers={**HEADERS,"Referer":"https://www.24h.com.vn/"},timeout=10); r.encoding="utf-8"
332
- m3u8s=re.findall(r'(https?://cdn\.24h\.com\.vn/[^\s"\'\\<>]+\.m3u8)', r.text)
333
- esc=[u.replace('\\//','/').replace('\\/','/') for u in re.findall(r'(https?:\\/\\/cdn\.24h\.com\.vn\\/[^\s"\'<>]+\.m3u8)', r.text)]
334
- all_urls=list(dict.fromkeys(m3u8s+esc))
335
- full=[u for u in all_urls if '_720p' not in u]
336
- p720=[u for u in all_urls if '_720p' in u]
337
  primary=full or p720
338
- if not primary: return None
339
- soup=BeautifulSoup(r.text,"lxml"); og=soup.find("meta",property="og:image")
340
- poster=og.get("content","") if og else ""
341
- base_url=primary[0]; parts=[base_url]; is_720p='_720p.m3u8' in base_url
342
- name=base_url.replace('_720p.m3u8','').replace('.m3u8','')
343
- m=re.search(r'(\D)(\d{1,2})$', name)
344
  if m:
345
- cur=int(m.group(2)); w=len(m.group(2)); bn=name[:m.start(2)]
346
- for i in range(cur+1, cur+5):
347
  nn=f"{bn}{i:02d}" if w==2 else f"{bn}{i}"
348
- for sfx in [('_720p.m3u8' if is_720p else '.m3u8'),'_720p.m3u8','.m3u8']:
349
  pu=nn+sfx
350
- if pu in parts: continue
351
  try:
352
- if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200: parts.append(pu); break
353
- except: continue
354
- else: break
355
  if len(parts)==1:
356
- m2=re.match(r'^(.+-)(\d{7,})(-.+?)(\d{1,2})((?:_720p)?\.m3u8)$', base_url)
357
  if m2:
358
  pfx,cid,mid,pn,sfx=m2.group(1),int(m2.group(2)),m2.group(3),int(m2.group(4)),m2.group(5)
359
  for i in range(1,5):
360
  pu=f"{pfx}{cid+i}{mid}{pn+i}{sfx}"
361
  try:
362
- if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200: parts.append(pu)
363
- else: break
364
- except: break
365
- if len(parts)>1: return {"src":parts[0],"poster":poster,"all_parts":parts}
366
- return {"src":primary[0],"poster":poster}
367
- except: return None
368
 
369
  def extract_bdp_video(url):
370
  try:
371
- m=re.search(r'-(\d{6,})\.html', url)
372
- if not m: return None
373
- r=requests.get(f"{BASE_BDP}/video-embed/{m.group(1)}.html",headers=HEADERS,timeout=10); r.encoding="utf-8"
374
- soup=BeautifulSoup(r.text,"lxml"); video=soup.select_one("video#videoPlayer")
375
- if not video: return None
376
  source=video.find("source")
377
- return {"src":source.get("src","") if source else "","poster":video.get("poster","")}
378
- except: return None
379
 
380
- # ═══ API Routes ═══
381
  VNE_CATS={"thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),"cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe")}
382
  BDP_CATS={"ngoai-hang-anh":("https://bongdaplus.vn/ngoai-hang-anh","Ngoại Hạng Anh"),"la-liga":("https://bongdaplus.vn/la-liga","La Liga"),"champions-league":("https://bongdaplus.vn/champions-league-cup-c1","Champions League"),"bong-da-vn":("https://bongdaplus.vn/bong-da-viet-nam","Bóng Đá VN")}
383
 
@@ -386,67 +247,77 @@ def api_homepage():
386
  def _f():
387
  articles=[]
388
  with ThreadPoolExecutor(6) as ex:
389
- futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in ["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri"]}
390
  futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
391
  for f in as_completed(futs):
392
  try:
393
- for a in f.result(): a["group"]=futs[f]; articles.append(a)
394
- except: pass
395
  try:
396
- for a in scrape_bdp(f"{BASE_BDP}/tin-moi")[:6]: a["group"]="Bóng Đá"; articles.append(a)
397
- except: pass
398
  return articles
399
  return JSONResponse(_cached("homepage",_f))
400
 
401
  @app.get("/api/category/{cat_id}")
402
  def api_category(cat_id:str):
403
  def _f():
404
- if cat_id=="bbc": return scrape_bbc_vietnamese()
405
- if cat_id in VNE_CATS:
406
- arts=scrape_vne(VNE_CATS[cat_id][0])
407
- for a in arts: a["group"]=VNE_CATS[cat_id][1]
408
- return arts
409
- if cat_id in BDP_CATS:
410
- arts=scrape_bdp(BDP_CATS[cat_id][0])
411
- for a in arts: a["group"]=BDP_CATS[cat_id][1]
412
- return arts
413
- return []
414
  return JSONResponse(_cached(f"cat_{cat_id}",_f))
415
 
416
  @app.get("/api/categories")
417
  def api_categories():
418
  cats=[{"id":"bbc","name":"BBC Tiếng Việt","source":"bbc"}]
419
- for k,(u,n) in VNE_CATS.items(): cats.append({"id":k,"name":n,"source":"vne"})
420
- for k,(u,n) in BDP_CATS.items(): cats.append({"id":k,"name":n,"source":"bdp"})
421
  return JSONResponse(cats)
422
 
423
  @app.get("/api/highlights")
424
- def api_highlights(): return JSONResponse(_cached("highlights",scrape_24h_highlights))
425
-
426
  @app.get("/api/shorts")
427
- def api_shorts(): return JSONResponse(_cached("shorts",scrape_24h_shorts))
428
-
429
  @app.get("/api/bdp_videos")
430
- def api_bdp_videos(): return JSONResponse(_cached("bdp_videos",scrape_bdp_videos))
431
 
432
  @app.get("/api/video_url")
433
  def api_video_url(url:str=Query(...)):
434
- if "24h.com.vn" in url: v=extract_video_url(url)
435
- elif "bongdaplus.vn" in url: v=extract_bdp_video(url)
436
- else: v=None
437
- return JSONResponse(v if v else {"error":"not found"})
438
 
439
  @app.get("/api/article")
440
  def api_article(url:str=Query(...)):
441
- if "vnexpress.net" in url: data=scrape_vne_article(url)
442
- elif "bbc.com" in url: data=scrape_bbc_article(url)
443
- else: data=None
444
- return JSONResponse(data if data else {"error":"not supported"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
445
 
446
- # ═══ Main page ═══
447
  @app.get("/")
448
  async def index():
449
- return FileResponse("/app/static/index.html")
450
 
451
- # Mount static
452
- app.mount("/static", StaticFiles(directory="/app/static"), name="static")
 
1
+ """VNEWS - FastAPI backend."""
2
+ import hashlib, re, time
3
  from concurrent.futures import ThreadPoolExecutor, as_completed
4
+ from fastapi import FastAPI, Query
5
+ from fastapi.responses import HTMLResponse, JSONResponse
6
  from fastapi.staticfiles import StaticFiles
7
  from urllib.parse import unquote, quote
8
  import requests
 
13
  BASE_BDP = "https://bongdaplus.vn"
14
  BASE_24H = "https://www.24h.com.vn"
15
  SPACE_URL = "https://bep40-vnews.hf.space"
16
+ _cache = {}
17
+ _cache_ttl = 300
18
 
19
+ def _cached(key, fn):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  now=time.time()
21
+ if key in _cache and now-_cache[key]["t"]<_cache_ttl:return _cache[key]["d"]
22
+ try:data=fn()
23
+ except:data=_cache.get(key,{}).get("d",[])
24
+ _cache[key]={"d":data,"t":now};return data
25
 
26
+ def _get(url,headers=None):
27
+ h=headers or HEADERS;r=requests.get(url,headers=h,timeout=15);r.encoding="utf-8"
28
+ return BeautifulSoup(r.text,"lxml")
 
29
 
30
  def scrape_vne(cat_url):
31
  try:
32
+ soup=_get(cat_url);arts=[]
33
  for it in soup.select("article.item-news")[:15]:
34
  a=it.select_one("h2.title-news a") or it.select_one("h3.title-news a")
35
+ if not a:continue
36
+ t=a.get("title","") or a.get_text(strip=True);lk=a.get("href","")
37
+ if not t or not lk:continue
38
+ im=it.find("img");img=(im.get("data-src") or im.get("src","")) if im else ""
39
+ if img and'blank'in img:
40
  src=it.find("source")
41
+ if src:img=src.get("srcset","").split(",")[0].strip().split(" ")[0]
42
  desc=it.select_one("p.description")
43
  arts.append({"title":t,"link":lk,"img":img,"summary":(desc.get_text(strip=True)[:150] if desc else ""),"source":"vne"})
44
  return arts
45
+ except:return[]
46
 
47
  def scrape_vne_article(url):
48
  try:
49
+ soup=_get(url);h1=soup.select_one("h1.title-detail");desc=soup.select_one("p.description")
50
+ og=soup.find("meta",property="og:image");og_img=og.get("content","") if og else ""
51
+ cd=soup.select_one("article.fck_detail");body=[]
52
  if cd:
53
  for ch in cd.children:
54
+ if not hasattr(ch,'name') or not ch.name:continue
55
+ if ch.name=="p":t=ch.get_text(strip=True);(body.append({"type":"p","text":t}) if t else None)
 
 
56
  elif ch.name=="figure":
57
  im=ch.find("img")
58
+ if im:s=im.get("data-src") or im.get("src","");cap=ch.find("figcaption");(body.append({"type":"img","src":s,"alt":cap.get_text(strip=True) if cap else ""}) if s else None)
59
+ elif ch.name in("h2","h3"):body.append({"type":"heading","text":ch.get_text(strip=True)})
60
+ return{"title":h1.get_text(strip=True) if h1 else "","summary":desc.get_text(strip=True) if desc else "","og_image":og_img,"body":body,"source":"vne","url":url}
61
+ except:return None
 
 
 
 
62
 
63
  def scrape_bdp(url):
64
  try:
65
+ soup=_get(url);arts=[];seen=set()
66
+ for sel in["div.news.fst","div.sld-itm.news","li.news"]:
67
  for it in soup.select(sel):
68
  tag=it.find("a",class_="title") or it.find("a",href=True)
69
+ if not tag:continue
70
+ t=tag.get_text(strip=True);lk=tag.get("href","")
71
+ if not t or len(t)<5:continue
72
+ if lk and not lk.startswith("http"):lk=BASE_BDP+lk
73
+ if lk in seen:continue
74
+ im=it.find("img");img=(im.get("data-src") or im.get("src","")) if im else ""
75
+ seen.add(lk);arts.append({"title":t,"link":lk,"img":img,"source":"bdp","summary":""})
76
  return arts[:15]
77
+ except:return[]
78
 
79
  def scrape_bdp_videos():
80
  try:
81
+ soup=_get(f"{BASE_BDP}/video");arts=[];seen=set()
82
  for a in soup.find_all("a",href=True):
83
  href=a.get("href","")
84
+ if"/video/" not in href or href in("/video/","/video/ban-thang-dep","/video/highlight"):continue
85
+ if not href.startswith("http"):href=BASE_BDP+href
86
+ if href in seen:continue
87
  title=re.sub(r'^\d{2}:\d{2}','',a.get_text(strip=True)).strip()
88
+ if not title or len(title)<5:continue
89
+ img_tag=a.find("img") or(a.parent.find("img") if a.parent else None)
90
  img=(img_tag.get("data-src") or img_tag.get("src","")) if img_tag else ""
91
+ seen.add(href);arts.append({"title":title,"link":href,"img":img,"source":"bdp"})
92
  return arts[:20]
93
+ except:return[]
94
 
95
  def scrape_24h_highlights():
96
  try:
97
+ soup=_get(f"{BASE_24H}/video-highlight-c953.html");arts=[];seen=set()
98
  for a in soup.find_all("a",href=True):
99
+ href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
100
+ if not title or len(title)<15 or"javascript:" in href:continue
101
+ if not href.startswith("http"):href=BASE_24H+href
102
+ if href in seen or"video" not in href.lower() or href.endswith("-c953.html"):continue
103
  seen.add(href)
104
+ img=a.find("img");img_src=""
105
+ if img:img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
106
+ if img_src and"base64" in img_src:img_src=""
107
  arts.append({"title":title,"link":href,"img":img_src,"source":"24h"})
108
  def _og(art):
109
+ if art["img"]:return
110
  try:
111
+ r=requests.get(art["link"],headers=HEADERS,timeout=8);r.encoding="utf-8"
112
  og=BeautifulSoup(r.text,"lxml").find("meta",property="og:image")
113
+ if og:art["img"]=og.get("content","")
114
+ except:pass
115
+ need=[a for a in arts if not a["img"]][:10]
116
  if need:
117
+ with ThreadPoolExecutor(5) as ex:list(ex.map(_og,need))
118
  return arts[:30]
119
+ except:return[]
120
 
121
  def scrape_24h_shorts():
122
  try:
123
+ r=requests.get(f"{BASE_24H}/video/video-tin-tuc-cvd769.html",headers=HEADERS,timeout=15);r.encoding="utf-8"
124
+ soup=BeautifulSoup(r.text,"lxml");arts=[];seen=set()
125
  for art in soup.find_all("article"):
126
  a=art.find("a",href=True)
127
+ if not a:continue
128
  href=a.get("href","")
129
+ if not href.startswith("http"):href=BASE_24H+href
130
+ if href in seen:continue
131
  img_tag=art.find("img")
132
  title=(img_tag.get("alt","") if img_tag else "") or a.get("title","") or a.get_text(strip=True)
133
+ if not title or len(title)<10:continue
134
  img_src=""
135
+ if img_tag:img_src=img_tag.get("data-original") or img_tag.get("data-src") or img_tag.get("src","")
136
+ if img_src and"base64" in img_src:img_src=""
137
+ seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
138
  if len(arts)<3:
139
  for a in soup.find_all("a",href=True):
140
  href=a.get("href","")
141
+ if not href.endswith(".html") or"javascript:" in href or"-cvd" in href:continue
142
+ if not href.startswith("http"):href=BASE_24H+href
143
+ if href in seen:continue
144
+ img=a.find("img") or(a.parent.find("img") if a.parent else None)
145
+ if not img:continue
146
  img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
147
+ if not img_src or"base64" in img_src:continue
148
  title=img.get("alt","") or a.get("title","") or a.get_text(strip=True)
149
+ if not title or len(title)<8:continue
150
+ seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
151
  return arts[:20]
152
+ except:return[]
153
 
154
  def scrape_bbc_vietnamese():
155
  try:
156
  bbc_h={"User-Agent":"Mozilla/5.0","Accept-Language":"en-GB,en;q=0.9"}
157
+ r=requests.get("https://www.bbc.com/vietnamese",headers=bbc_h,timeout=15);r.encoding="utf-8"
158
+ soup=BeautifulSoup(r.text,"lxml");arts=[];seen=set()
159
  for a in soup.select("a[href*='/vietnamese/']"):
160
  href=a.get("href","")
161
+ if not href or href=="/vietnamese" or href.count("/")<3:continue
162
+ if not href.startswith("http"):href="https://www.bbc.com"+href
163
+ if href in seen:continue
164
  title=a.get_text(strip=True)
165
+ if not title or len(title)<15:continue
166
+ if any(x in title.lower() for x in["đăng nhập","trang chủ","bbc news"]):continue
167
+ img="";container=a.parent
168
  for _ in range(3):
169
  if container:
170
  im=container.find("img")
171
+ if im:img=im.get("src","") or im.get("data-src","");break
 
 
172
  container=container.parent
173
+ seen.add(href);arts.append({"title":title,"link":href,"img":img,"source":"bbc","summary":""})
174
+ if len(arts)>=15:break
175
  return arts
176
+ except:return[]
177
 
178
  def scrape_bbc_article(url):
179
  try:
180
+ r=requests.get(url,headers={"User-Agent":"Mozilla/5.0","Accept-Language":"en-GB"},timeout=15);r.encoding="utf-8"
181
+ soup=BeautifulSoup(r.text,"lxml");h1=soup.find("h1")
182
+ og=soup.find("meta",property="og:image");og_img=og.get("content","") if og else ""
183
  body=[]
184
  for p in soup.select("[data-component='text-block'] p, article p, main p"):
185
  t=p.get_text(strip=True)
186
+ if t and len(t)>20:body.append({"type":"p","text":t})
187
+ for img in soup.select("main img, article img"):
188
+ src=img.get("src","")
189
+ if src and("ichef" in src or"bbci" in src):body.append({"type":"img","src":src,"alt":img.get("alt","")})
190
+ return{"title":h1.get_text(strip=True) if h1 else "","summary":"","og_image":og_img,"body":body,"source":"bbc","url":url}
191
+ except:return None
192
 
193
  def extract_video_url(article_url):
194
  try:
195
+ r=requests.get(article_url,headers={**HEADERS,"Referer":"https://www.24h.com.vn/"},timeout=10);r.encoding="utf-8"
196
+ m3u8s=re.findall(r'(https?://cdn\.24h\.com\.vn/[^\s"\'\\<>]+\.m3u8)',r.text)
197
+ esc=[u.replace('\\//','/').replace('\\/','/') for u in re.findall(r'(https?:\\/\\/cdn\.24h\.com\.vn\\/[^\s"\'<>]+\.m3u8)',r.text)]
198
+ all_urls=list(dict.fromkeys(m3u8s+esc));full=[u for u in all_urls if'_720p' not in u];p720=[u for u in all_urls if'_720p' in u]
 
 
199
  primary=full or p720
200
+ if not primary:return None
201
+ soup=BeautifulSoup(r.text,"lxml");og=soup.find("meta",property="og:image");poster=og.get("content","") if og else ""
202
+ base_url=primary[0];parts=[base_url]
203
+ is_720p='_720p.m3u8' in base_url;name=base_url.replace('_720p.m3u8','').replace('.m3u8','')
204
+ m=re.search(r'(\D)(\d{1,2})$',name)
 
205
  if m:
206
+ cur=int(m.group(2));w=len(m.group(2));bn=name[:m.start(2)]
207
+ for i in range(cur+1,cur+5):
208
  nn=f"{bn}{i:02d}" if w==2 else f"{bn}{i}"
209
+ for sfx in[('_720p.m3u8' if is_720p else '.m3u8'),'_720p.m3u8','.m3u8']:
210
  pu=nn+sfx
211
+ if pu in parts:continue
212
  try:
213
+ if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200:parts.append(pu);break
214
+ except:continue
215
+ else:break
216
  if len(parts)==1:
217
+ m2=re.match(r'^(.+-)(\d{7,})(-.+?)(\d{1,2})((?:_720p)?\.m3u8)$',base_url)
218
  if m2:
219
  pfx,cid,mid,pn,sfx=m2.group(1),int(m2.group(2)),m2.group(3),int(m2.group(4)),m2.group(5)
220
  for i in range(1,5):
221
  pu=f"{pfx}{cid+i}{mid}{pn+i}{sfx}"
222
  try:
223
+ if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200:parts.append(pu)
224
+ else:break
225
+ except:break
226
+ if len(parts)>1:return{"src":parts[0],"poster":poster,"all_parts":parts}
227
+ return{"src":primary[0],"poster":poster}
228
+ except:return None
229
 
230
  def extract_bdp_video(url):
231
  try:
232
+ m=re.search(r'-(\d{6,})\.html',url)
233
+ if not m:return None
234
+ r=requests.get(f"{BASE_BDP}/video-embed/{m.group(1)}.html",headers=HEADERS,timeout=10);r.encoding="utf-8"
235
+ soup=BeautifulSoup(r.text,"lxml");video=soup.select_one("video#videoPlayer")
236
+ if not video:return None
237
  source=video.find("source")
238
+ return{"src":source.get("src","") if source else "","poster":video.get("poster","")}
239
+ except:return None
240
 
241
+ # API
242
  VNE_CATS={"thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),"cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe")}
243
  BDP_CATS={"ngoai-hang-anh":("https://bongdaplus.vn/ngoai-hang-anh","Ngoại Hạng Anh"),"la-liga":("https://bongdaplus.vn/la-liga","La Liga"),"champions-league":("https://bongdaplus.vn/champions-league-cup-c1","Champions League"),"bong-da-vn":("https://bongdaplus.vn/bong-da-viet-nam","Bóng Đá VN")}
244
 
 
247
  def _f():
248
  articles=[]
249
  with ThreadPoolExecutor(6) as ex:
250
+ futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri"]}
251
  futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
252
  for f in as_completed(futs):
253
  try:
254
+ for a in f.result():a["group"]=futs[f];articles.append(a)
255
+ except:pass
256
  try:
257
+ for a in scrape_bdp(f"{BASE_BDP}/tin-moi")[:6]:a["group"]="Bóng Đá";articles.append(a)
258
+ except:pass
259
  return articles
260
  return JSONResponse(_cached("homepage",_f))
261
 
262
  @app.get("/api/category/{cat_id}")
263
  def api_category(cat_id:str):
264
  def _f():
265
+ if cat_id=="bbc":return scrape_bbc_vietnamese()
266
+ if cat_id in VNE_CATS:arts=scrape_vne(VNE_CATS[cat_id][0]);[a.update({"group":VNE_CATS[cat_id][1]}) for a in arts];return arts
267
+ if cat_id in BDP_CATS:arts=scrape_bdp(BDP_CATS[cat_id][0]);[a.update({"group":BDP_CATS[cat_id][1]}) for a in arts];return arts
268
+ return[]
 
 
 
 
 
 
269
  return JSONResponse(_cached(f"cat_{cat_id}",_f))
270
 
271
  @app.get("/api/categories")
272
  def api_categories():
273
  cats=[{"id":"bbc","name":"BBC Tiếng Việt","source":"bbc"}]
274
+ for k,(u,n) in VNE_CATS.items():cats.append({"id":k,"name":n,"source":"vne"})
275
+ for k,(u,n) in BDP_CATS.items():cats.append({"id":k,"name":n,"source":"bdp"})
276
  return JSONResponse(cats)
277
 
278
  @app.get("/api/highlights")
279
+ def api_highlights():return JSONResponse(_cached("highlights",scrape_24h_highlights))
 
280
  @app.get("/api/shorts")
281
+ def api_shorts():return JSONResponse(_cached("shorts",scrape_24h_shorts))
 
282
  @app.get("/api/bdp_videos")
283
+ def api_bdp_videos():return JSONResponse(_cached("bdp_videos",scrape_bdp_videos))
284
 
285
  @app.get("/api/video_url")
286
  def api_video_url(url:str=Query(...)):
287
+ if"24h.com.vn" in url:v=extract_video_url(url)
288
+ elif"bongdaplus.vn" in url:v=extract_bdp_video(url)
289
+ else:v=None
290
+ return JSONResponse(v if v else{"error":"not found"})
291
 
292
  @app.get("/api/article")
293
  def api_article(url:str=Query(...)):
294
+ if"vnexpress.net" in url:data=scrape_vne_article(url)
295
+ elif"bbc.com" in url:data=scrape_bbc_article(url)
296
+ else:data=None
297
+ return JSONResponse(data if data else{"error":"not supported"})
298
+
299
+ # Share page: SEO with og:image + redirect to article
300
+ @app.get("/s")
301
+ async def share_redirect(url:str=Query(default=""),title:str=Query(default="VNEWS"),img:str=Query(default="")):
302
+ """Share URL with proper og:image. Format: /s?url=ENCODED_ARTICLE_URL&title=...&img=..."""
303
+ og_image=unquote(img) if img else "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
304
+ decoded_url=unquote(url)
305
+ redirect_script=f'<script>localStorage.setItem("pending_article","{decoded_url}");location.href="{SPACE_URL}";</script>' if decoded_url else f'<script>location.href="{SPACE_URL}";</script>'
306
+ return HTMLResponse(f'''<!DOCTYPE html><html><head>
307
+ <meta charset="utf-8"><title>{unquote(title)}</title>
308
+ <meta property="og:title" content="{unquote(title)}">
309
+ <meta property="og:image" content="{og_image}">
310
+ <meta property="og:url" content="{SPACE_URL}/s?url={url}&title={title}&img={img}">
311
+ <meta property="og:type" content="article">
312
+ <meta property="og:site_name" content="VNEWS">
313
+ <meta property="og:description" content="{unquote(title)} - Đọc trên VNEWS">
314
+ <meta name="twitter:card" content="summary_large_image">
315
+ <meta name="twitter:title" content="{unquote(title)}">
316
+ <meta name="twitter:image" content="{og_image}">
317
+ </head><body>{redirect_script}</body></html>''')
318
 
 
319
  @app.get("/")
320
  async def index():
321
+ with open("/app/static/index.html","r",encoding="utf-8") as f:return HTMLResponse(content=f.read())
322
 
323
+ app.mount("/static",StaticFiles(directory="/app/static"),name="static")