bep40 commited on
Commit
76b264c
·
verified ·
1 Parent(s): 6003e2e

Replace vnreview with genk.vn AI (articles readable in-app via .knc-content parser)

Browse files
Files changed (1) hide show
  1. main.py +39 -23
main.py CHANGED
@@ -340,31 +340,30 @@ def scrape_dantri_congnghe():
340
  if len(arts)>=15:break
341
  return arts
342
  except:return[]
343
- def scrape_vnreview_ai():
344
- """Scrape AI articles from vnreview.vn"""
345
  try:
346
- r=requests.get("https://vnreview.vn/chuyen-muc/ai-pho-thong.28/",headers=HEADERS,timeout=15)
347
  if r.status_code!=200:return[]
348
  r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
349
- import re as _re
350
  articles=[];seen=set()
351
- for img in soup.find_all("img"):
352
- src=img.get("src","") or img.get("data-src","")
353
- if "cdn.vnreview.vn" not in src or "avatars" in src or "logo" in src:continue
354
- parent=img.parent;link=None
355
- for _ in range(5):
356
- if parent is None:break
357
- link=parent.find("a",href=_re.compile(r"/threads/"))
358
- if link:break
359
- parent=parent.parent
360
- if not link:continue
361
- href=link.get("href","")
362
- if not href.startswith("http"):href="https://vnreview.vn"+href
363
- if href in seen:continue
364
- title=link.get("title","") or link.get_text(strip=True) or img.get("alt","")
365
- if not title or len(title)<10:continue
366
- seen.add(href);articles.append({"title":title,"link":href,"img":src,"source":"vnreview"})
367
- if len(articles)>=12:break
368
  return articles
369
  except:return[]
370
 
@@ -455,13 +454,30 @@ def api_categories():
455
  return JSONResponse(cats)
456
  @app.get("/api/dantri_hot")
457
  def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
458
- @app.get("/api/vnreview_ai")
459
- def api_vnreview_ai():return JSONResponse(_cached("vnreview_ai",scrape_vnreview_ai,ttl=_cache_ttl))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
460
  @app.get("/api/article")
461
  def api_article(url:str=Query(...)):
462
  if"vnexpress.net" in url:data=scrape_vne_article(url)
463
  elif"bbc.com" in url:data=scrape_bbc_article(url)
464
  elif"dantri.com.vn" in url:data=scrape_dantri_article(url)
 
465
  else:data=None
466
  return JSONResponse(data if data else{"error":"not supported"})
467
  @app.get("/v")
 
340
  if len(arts)>=15:break
341
  return arts
342
  except:return[]
343
+ def scrape_genk_ai():
344
+ """Scrape AI articles from genk.vn - readable in-app"""
345
  try:
346
+ r=requests.get("https://genk.vn/ai.chn",headers=HEADERS,timeout=15)
347
  if r.status_code!=200:return[]
348
  r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
 
349
  articles=[];seen=set()
350
+ for a in soup.find_all("a",href=True):
351
+ href=a.get("href","")
352
+ if not href.endswith(".chn") or href=="/ai.chn":continue
353
+ if href.startswith("/"):href="https://genk.vn"+href
354
+ if href in seen or "genk.vn" not in href:continue
355
+ title=a.get("title","") or a.get_text(strip=True)
356
+ if not title or len(title)<20:continue
357
+ container=a.parent
358
+ img_src=""
359
+ for _ in range(4):
360
+ if container is None:break
361
+ img=container.find("img",attrs={"data-src":True})
362
+ if not img:img=container.find("img",src=lambda s:s and "mediacdn" in s)
363
+ if img:img_src=img.get("data-src","") or img.get("src","");break
364
+ container=container.parent
365
+ seen.add(href);articles.append({"title":title,"link":href,"img":img_src,"source":"genk"})
366
+ if len(articles)>=15:break
367
  return articles
368
  except:return[]
369
 
 
454
  return JSONResponse(cats)
455
  @app.get("/api/dantri_hot")
456
  def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
457
+ @app.get("/api/genk_ai")
458
+ def api_genk_ai():return JSONResponse(_cached("genk_ai",scrape_genk_ai,ttl=_cache_ttl))
459
+ def scrape_genk_article(url):
460
+ try:
461
+ r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
462
+ h1=soup.find("h1");og=soup.find("meta",property="og:image");og_img=og.get("content","") if og else ""
463
+ desc_el=soup.find("meta",property="og:description");desc=desc_el.get("content","") if desc_el else ""
464
+ cd=soup.select_one(".knc-content");body=[]
465
+ if cd:
466
+ for el in cd.find_all(["p","h2","h3","figure","img"],recursive=True):
467
+ if el.name=="p":t=el.get_text(strip=True);(body.append({"type":"p","text":t}) if t and len(t)>15 else None)
468
+ elif el.name in("h2","h3"):t=el.get_text(strip=True);(body.append({"type":"heading","text":t}) if t else None)
469
+ elif el.name in("figure","img"):
470
+ im=el if el.name=="img" else el.find("img")
471
+ if im:s=im.get("data-src") or im.get("src","");(body.append({"type":"img","src":s}) if s and"base64" not in s else None)
472
+ return{"title":h1.get_text(strip=True) if h1 else "","summary":desc,"og_image":og_img,"body":body,"source":"genk","url":url}
473
+ except:return None
474
+
475
  @app.get("/api/article")
476
  def api_article(url:str=Query(...)):
477
  if"vnexpress.net" in url:data=scrape_vne_article(url)
478
  elif"bbc.com" in url:data=scrape_bbc_article(url)
479
  elif"dantri.com.vn" in url:data=scrape_dantri_article(url)
480
+ elif"genk.vn" in url:data=scrape_genk_article(url)
481
  else:data=None
482
  return JSONResponse(data if data else{"error":"not supported"})
483
  @app.get("/v")