Spaces:
Running
Running
Replace vnreview with genk.vn AI (articles readable in-app via .knc-content parser)
Browse files
main.py
CHANGED
|
@@ -340,31 +340,30 @@ def scrape_dantri_congnghe():
|
|
| 340 |
if len(arts)>=15:break
|
| 341 |
return arts
|
| 342 |
except:return[]
|
| 343 |
-
def
|
| 344 |
-
"""Scrape AI articles from
|
| 345 |
try:
|
| 346 |
-
r=requests.get("https://
|
| 347 |
if r.status_code!=200:return[]
|
| 348 |
r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
| 349 |
-
import re as _re
|
| 350 |
articles=[];seen=set()
|
| 351 |
-
for
|
| 352 |
-
|
| 353 |
-
if
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
seen.add(href);articles.append({"title":title,"link":href,"img":
|
| 367 |
-
if len(articles)>=
|
| 368 |
return articles
|
| 369 |
except:return[]
|
| 370 |
|
|
@@ -455,13 +454,30 @@ def api_categories():
|
|
| 455 |
return JSONResponse(cats)
|
| 456 |
@app.get("/api/dantri_hot")
|
| 457 |
def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
|
| 458 |
-
@app.get("/api/
|
| 459 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 460 |
@app.get("/api/article")
|
| 461 |
def api_article(url:str=Query(...)):
|
| 462 |
if"vnexpress.net" in url:data=scrape_vne_article(url)
|
| 463 |
elif"bbc.com" in url:data=scrape_bbc_article(url)
|
| 464 |
elif"dantri.com.vn" in url:data=scrape_dantri_article(url)
|
|
|
|
| 465 |
else:data=None
|
| 466 |
return JSONResponse(data if data else{"error":"not supported"})
|
| 467 |
@app.get("/v")
|
|
|
|
| 340 |
if len(arts)>=15:break
|
| 341 |
return arts
|
| 342 |
except:return[]
|
| 343 |
+
def scrape_genk_ai():
|
| 344 |
+
"""Scrape AI articles from genk.vn - readable in-app"""
|
| 345 |
try:
|
| 346 |
+
r=requests.get("https://genk.vn/ai.chn",headers=HEADERS,timeout=15)
|
| 347 |
if r.status_code!=200:return[]
|
| 348 |
r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
|
|
|
| 349 |
articles=[];seen=set()
|
| 350 |
+
for a in soup.find_all("a",href=True):
|
| 351 |
+
href=a.get("href","")
|
| 352 |
+
if not href.endswith(".chn") or href=="/ai.chn":continue
|
| 353 |
+
if href.startswith("/"):href="https://genk.vn"+href
|
| 354 |
+
if href in seen or "genk.vn" not in href:continue
|
| 355 |
+
title=a.get("title","") or a.get_text(strip=True)
|
| 356 |
+
if not title or len(title)<20:continue
|
| 357 |
+
container=a.parent
|
| 358 |
+
img_src=""
|
| 359 |
+
for _ in range(4):
|
| 360 |
+
if container is None:break
|
| 361 |
+
img=container.find("img",attrs={"data-src":True})
|
| 362 |
+
if not img:img=container.find("img",src=lambda s:s and "mediacdn" in s)
|
| 363 |
+
if img:img_src=img.get("data-src","") or img.get("src","");break
|
| 364 |
+
container=container.parent
|
| 365 |
+
seen.add(href);articles.append({"title":title,"link":href,"img":img_src,"source":"genk"})
|
| 366 |
+
if len(articles)>=15:break
|
| 367 |
return articles
|
| 368 |
except:return[]
|
| 369 |
|
|
|
|
| 454 |
return JSONResponse(cats)
|
| 455 |
@app.get("/api/dantri_hot")
|
| 456 |
def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
|
| 457 |
+
@app.get("/api/genk_ai")
|
| 458 |
+
def api_genk_ai():return JSONResponse(_cached("genk_ai",scrape_genk_ai,ttl=_cache_ttl))
|
| 459 |
+
def scrape_genk_article(url):
|
| 460 |
+
try:
|
| 461 |
+
r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
| 462 |
+
h1=soup.find("h1");og=soup.find("meta",property="og:image");og_img=og.get("content","") if og else ""
|
| 463 |
+
desc_el=soup.find("meta",property="og:description");desc=desc_el.get("content","") if desc_el else ""
|
| 464 |
+
cd=soup.select_one(".knc-content");body=[]
|
| 465 |
+
if cd:
|
| 466 |
+
for el in cd.find_all(["p","h2","h3","figure","img"],recursive=True):
|
| 467 |
+
if el.name=="p":t=el.get_text(strip=True);(body.append({"type":"p","text":t}) if t and len(t)>15 else None)
|
| 468 |
+
elif el.name in("h2","h3"):t=el.get_text(strip=True);(body.append({"type":"heading","text":t}) if t else None)
|
| 469 |
+
elif el.name in("figure","img"):
|
| 470 |
+
im=el if el.name=="img" else el.find("img")
|
| 471 |
+
if im:s=im.get("data-src") or im.get("src","");(body.append({"type":"img","src":s}) if s and"base64" not in s else None)
|
| 472 |
+
return{"title":h1.get_text(strip=True) if h1 else "","summary":desc,"og_image":og_img,"body":body,"source":"genk","url":url}
|
| 473 |
+
except:return None
|
| 474 |
+
|
| 475 |
@app.get("/api/article")
|
| 476 |
def api_article(url:str=Query(...)):
|
| 477 |
if"vnexpress.net" in url:data=scrape_vne_article(url)
|
| 478 |
elif"bbc.com" in url:data=scrape_bbc_article(url)
|
| 479 |
elif"dantri.com.vn" in url:data=scrape_dantri_article(url)
|
| 480 |
+
elif"genk.vn" in url:data=scrape_genk_article(url)
|
| 481 |
else:data=None
|
| 482 |
return JSONResponse(data if data else{"error":"not supported"})
|
| 483 |
@app.get("/v")
|