Update app/fetch.py

app/fetch.py

@@ -1,22 +1,21 @@
 import random
 import json
-import asyncio
 from fastapi import HTTPException
 from playwright.async_api import async_playwright, TimeoutError
 
-
 UAS = [
-    # Windows Chrome
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
     "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-
-    # macOS Chrome
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
     "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
 ]
 
+PROXY = {
+    "server": "http://proxy.scrapingant.com:8080",
+    "username": "scrapingant&proxy_type=residential&proxy_country=VN&browser=false",
+    "password": "fdd15bd8d6f042f893d8bb93a8e47722",
+}
+
 
 class Fetcher:
     def __init__(self):
@@ -27,6 +26,7 @@ class Fetcher:
         self.pw = await async_playwright().start()
         self.browser = await self.pw.chromium.launch(
             headless=True,
+            proxy=PROXY,
             args=[
                 "--no-sandbox",
                 "--disable-dev-shm-usage",
@@ -40,10 +40,8 @@ class Fetcher:
         if self.pw:
             await self.pw.stop()
 
-    # =============================
-
-    async def _new_context(self):
-        return await self.browser.new_context(
+    async def fetch(self, url: str):
+        context = await self.browser.new_context(
             user_agent=random.choice(UAS),
             locale="en-US",
             timezone_id="Asia/Ho_Chi_Minh",
@@ -53,45 +51,32 @@ class Fetcher:
             },
         )
 
-    async def fetch(self, url: str, retry: int = 1):
-        context = await self._new_context()
         page = await context.new_page()
 
         try:
-            # ===== CF WARM UP =====
+            # warm CF
             await page.goto(
                 "https://ihentai.dog/",
                 wait_until="domcontentloaded",
-                timeout=15000,
+                timeout=20000,
             )
-            await page.wait_for_timeout(1200)  # CF settle
+            await page.wait_for_timeout(1500)
 
-            # ===== REAL NAVIGATION =====
             resp = await page.goto(
                 url,
                 wait_until="domcontentloaded",
-                timeout=15000,
+                timeout=20000,
             )
 
             if not resp:
                 raise HTTPException(500, "No response")
 
             status = resp.status
-            headers = resp.headers
             body = await resp.text()
 
-            # ===== STATUS HANDLE =====
             if status >= 400:
                 raise HTTPException(status, "Upstream error")
 
-            if not body:
-                return {
-                    "status": status,
-                    "headers": headers,
-                    "data": None,
-                }
-
-            # ===== JSON SAFE PARSE =====
             try:
                 data = json.loads(body)
             except Exception:
@@ -99,15 +84,10 @@ class Fetcher:
 
             return {
                 "status": status,
-                "headers": headers,
                 "data": data,
             }
 
         except TimeoutError:
-            if retry > 0:
-                await page.close()
-                await context.close()
-                return await self.fetch(url, retry=retry - 1)
             raise HTTPException(504, "Timeout")
 
         finally: