fix: rate limiter + session pooling + cache for all scrape functions

main.py

@@ -25,6 +25,20 @@ _cache_ttl = 300
 _cache_ttl_live = 60
 _cache_ttl_yt = 1800
 
+# ===== Connection Pool Session (reuse TCP connections) =====
+_session_pool = threading.local()
+def _get_session():
+    """Get a thread-local requests.Session with connection pooling."""
+    s = getattr(_session_pool, "sess", None)
+    if s is None:
+        s = requests.Session()
+        s.headers.update(HEADERS)
+        adapter = requests.adapters.HTTPAdapter(pool_connections=6, pool_maxsize=6, max_retries=1)
+        s.mount("http://", adapter)
+        s.mount("https://", adapter)
+        _session_pool.sess = s
+    return s
+
 # ===== VTV NAM BO SHORTS FALLBACK =====
 SHORTS_FALLBACK = [
     {"id":"nqlLH6chLRo","title":"Tin nóng VTV Nam Bộ | #shorts","channel":"vtvnambo"},
@@ -97,20 +111,21 @@ class _RateLimiter:
                 time.sleep(wait_time)
             self._last[domain] = time.time()
 
-_rl_bongda = _RateLimiter(min_interval=2.0)
-_rl_news  = _RateLimiter(min_interval=0.5)
+_rl_bongda = _RateLimiter(min_interval=3.0)
+_rl_news  = _RateLimiter(min_interval=2.0)
 
 def _rl_get(url, headers=None, timeout=10, domain=None):
     from urllib.parse import urlparse
     if domain is None:
         domain = urlparse(url).netloc
     rl = _rl_bongda if "bongda.com.vn" in domain else _rl_news
+    sess = _get_session()
     for attempt in range(3):
         rl.wait(domain)
         try:
-            r = requests.get(url, headers=headers, timeout=timeout)
+            r = sess.get(url, headers=headers, timeout=timeout)
             if r.status_code == 429:
-                retry_after = int(r.headers.get("Retry-After", 5))
+                retry_after = int(r.headers.get("Retry-After", 10))
                 print(f"[RateLimit] 429 from {domain}, waiting {retry_after}s")
                 time.sleep(retry_after)
                 continue
@@ -146,7 +161,7 @@ def _cached(key, fn, ttl=None):
     except: data=_cache.get(key,{}).get("d",[])
     _cache[key]={"d":data,"t":now}; return data
 def _get(url, headers=None):
-    h=headers or HEADERS; r=requests.get(url, headers=h, timeout=15); r.encoding="utf-8"
+    h=headers or HEADERS; r=_get_session().get(url, headers=h, timeout=15); r.encoding="utf-8"
     return BeautifulSoup(r.text,"lxml")
 def fetch_bongda_api(endpoint):
     try:
@@ -517,8 +532,8 @@ def scrape_vne(cat_url):
 
 def scrape_genk_ai():
     try:
-        r=requests.get("https://genk.vn/ai.chn",headers=HEADERS,timeout=15)
-        if r.status_code!=200: return []
+        r=_rl_get("https://genk.vn/ai.chn", headers=HEADERS, timeout=15, domain="genk.vn")
+        if not r or r.status_code!=200: return []
         r.encoding="utf-8"; soup=BeautifulSoup(r.text,"lxml"); articles=[]; seen=set()
         for a in soup.find_all("a",href=True):
             href=a.get("href","")
@@ -538,9 +553,11 @@ def scrape_genk_ai():
             seen.add(href)
             if not img_src:
                 try:
-                    og_r=requests.get(href,headers=HEADERS,timeout=8); og_r.encoding="utf-8"
-                    og_soup=BeautifulSoup(og_r.text,"lxml"); og_tag=og_soup.find("meta",property="og:image")
-                    if og_tag: img_src=og_tag.get("content","")
+                    og_r=_rl_get(href, headers=HEADERS, timeout=8, domain="genk.vn")
+                    if og_r and og_r.status_code==200:
+                        og_r.encoding="utf-8"
+                        og_soup=BeautifulSoup(og_r.text,"lxml"); og_tag=og_soup.find("meta",property="og:image")
+                        if og_tag: img_src=og_tag.get("content","")
                 except: pass
             articles.append({"title":title,"link":href,"img":img_src,"source":"genk"})
             if len(articles)>=30: break
@@ -551,14 +568,17 @@ def scrape_genk_ai():
 def api_homepage():
     def _f():
         articles=[]
-        with ThreadPoolExecutor(12) as ex:
+        # Reduced concurrency to avoid HF outbound rate limits
+        with ThreadPoolExecutor(4) as ex:
             futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri","phap-luat","giao-duc","du-lich","doi-song"]}
             for f in as_completed(futs):
                 try:
                     for a in f.result(): a["group"]=futs[f]; articles.append(a)
                 except: pass
         return articles
-    return JSONResponse(_cached("homepage",_f))
+    resp = JSONResponse(_cached("homepage",_f))
+    resp.headers["Cache-Control"] = "public, max-age=120"
+    return resp
 
 @app.get("/api/category/{cat_id}")
 def api_category(cat_id:str):
@@ -569,7 +589,9 @@ def api_category(cat_id:str):
             [a.update({"group":VNE_CATS[cat_id][1]}) for a in arts]
             return arts
         return []
-    return JSONResponse(_cached(f"cat_{cat_id}",_f))
+    resp = JSONResponse(_cached(f"cat_{cat_id}",_f))
+    resp.headers["Cache-Control"] = "public, max-age=120"
+    return resp
 
 @app.get("/api/categories")
 def api_categories():
@@ -581,8 +603,11 @@ def api_categories():
 # ===== Article scraping functions for /api/article =====
 def _scrape_generic(url):
     """Generic article scraper fallback."""
+    from urllib.parse import urlparse
     try:
-        r=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,allow_redirects=True)
+        domain=urlparse(url).netloc
+        r=_rl_get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,domain=domain)
+        if not r: return None
         r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript']):tag.decompose()
         h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
@@ -622,7 +647,8 @@ def scrape_vne_article(url):
         return data
     # If generic didn't work well, try specific VnExpress selectors
     try:
-        r=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15)
+        r=_rl_get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,domain="vnexpress.net")
+        if not r: return data
         r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','.banner-ads','iframe','.fb-comments','.fb-root']):tag.decompose()
         article=soup.select_one('.fck_detail,.sidebar-1')
@@ -668,7 +694,8 @@ def scrape_dantri_article(url):
     """Scrape Dân Trí article."""
     if 'dantri.com.vn' not in url:return _scrape_generic(url)
     try:
-        r=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15)
+        r=_rl_get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,domain="dantri.com.vn")
+        if not r: return _scrape_generic(url)
         r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','.banner-ads','iframe']):tag.decompose()
         h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
@@ -712,7 +739,8 @@ def scrape_genk_article(url):
     """Scrape GenK article."""
     if 'genk.vn' not in url:return _scrape_generic(url)
     try:
-        r=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15)
+        r=_rl_get(url,headers={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,domain="genk.vn")
+        if not r: return _scrape_generic(url)
         r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','.ads','iframe']):tag.decompose()
         h1=soup.find('h1');ogt=soup.find('meta',property='og:title')