bep40 commited on
Commit
366c0ff
·
verified ·
1 Parent(s): 5faf298

Fix genk.vn scraper: search 6 levels up for images, filter avatars/logos

Browse files
Files changed (1) hide show
  1. main.py +7 -6
main.py CHANGED
@@ -354,13 +354,14 @@ def scrape_genk_ai():
354
  if href in seen or "genk.vn" not in href:continue
355
  title=a.get("title","") or a.get_text(strip=True)
356
  if not title or len(title)<20:continue
357
- container=a.parent
358
- img_src=""
359
- for _ in range(4):
360
  if container is None:break
361
- img=container.find("img",attrs={"data-src":True})
362
- if not img:img=container.find("img",src=lambda s:s and "mediacdn" in s)
363
- if img:img_src=img.get("data-src","") or img.get("src","");break
 
 
364
  container=container.parent
365
  seen.add(href);articles.append({"title":title,"link":href,"img":img_src,"source":"genk"})
366
  if len(articles)>=15:break
 
354
  if href in seen or "genk.vn" not in href:continue
355
  title=a.get("title","") or a.get_text(strip=True)
356
  if not title or len(title)<20:continue
357
+ container=a.parent;img_src=""
358
+ for _ in range(6):
 
359
  if container is None:break
360
+ for img in container.find_all("img"):
361
+ s=img.get("data-src","") or img.get("src","")
362
+ if s and "mediacdn" in s and "avatar" not in s and "logo" not in s:
363
+ img_src=s;break
364
+ if img_src:break
365
  container=container.parent
366
  seen.add(href);articles.append({"title":title,"link":href,"img":img_src,"source":"genk"})
367
  if len(articles)>=15:break