Update scraper.py

scraper.py

@@ -1,28 +1,35 @@
 import asyncio
 import random
 import re
-from curl_cffi.requests import AsyncSession
+import aiohttp
 from logger import get_logger
 
 log = get_logger()
 
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0 Safari/537.36"
+]
+
 async def check_fragment(word: str, proxy_url: str = None) -> str:
     url = f"https://fragment.com/username/{word}"
-    proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
-
-    try:
-        async with AsyncSession(impersonate="chrome120", proxies=proxies) as session:
-            for attempt in range(3):
-                try:
-                    resp = await session.get(url, timeout=15, allow_redirects=True)
+    headers = {"User-Agent": random.choice(USER_AGENTS)}
 
-                    if resp.status_code in [429, 403]:
-                        log.warning(f"Rate limited on {word}. Retrying...")
+    for attempt in range(3):
+        try:
+            # We open a fresh session per attempt to prevent dead proxies from leaking connections
+            timeout = aiohttp.ClientTimeout(total=15)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(url, headers=headers, proxy=proxy_url, allow_redirects=True) as resp:
+                    
+                    if resp.status in [429, 403]:
                         await asyncio.sleep(random.uniform(2.0, 4.0))
                         continue
 
-                    html = resp.text
+                    html = await resp.text()
 
+                    # Cloudflare block
                     if "Just a moment..." in html or "cloudflare" in html.lower():
                         await asyncio.sleep(random.uniform(2, 4))
                         continue
@@ -42,32 +49,21 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
                         if "auction" in s: return "ON_AUCTION"
                         if "available" in s: return "AVAILABLE"
                         if "sale" in s or "purchase" in s: return "FOR_SALE"
+                        return s.upper()
 
-                        # If it found a status span but we don't recognize the text
-                        clean = s.upper()
-                        log.warning(f"UNKNOWN STATUS TEXT for {word}: '{clean}'")
-                        return clean
-
-                    # 3. FALLBACK HTML PARSING (Fixing the False Unavailables)
+                    # 3. FALLBACK HTML PARSING
                     if 'class="tm-status-taken"' in html: return "TAKEN"
                     
-                    # 'tm-status-unavail' is used for BOTH Sold and Unavailable. We must differentiate.
                     if 'class="tm-status-unavail"' in html:
                         if ">Sold<" in html or ">sold<" in html.lower() or "recently sold" in html.lower():
                             return "SOLD"
                         return "UNAVAILABLE"
 
-                    # If absolutely no markers exist, it might be a new UI layout.
-                    log.error(f"FAILED TO PARSE UI FOR: {word}. HTML snippet logged.")
-                    with open("failed_parse.log", "a", encoding="utf-8") as f:
-                        f.write(f"\n--- {word} ---\n{html[:1000]}\n")
                     return "ERROR"
 
-                except Exception as e:
-                    log.error(f"Request error for {word} (Attempt {attempt+1}): {str(e)}")
-                    await asyncio.sleep(1.5)
-
-    except Exception as e:
-        log.error(f"Session error for {word}: {str(e)}")
+        except Exception as e:
+            # This logs the specific aiohttp timeout/connection error so you can see if a proxy dies
+            log.error(f"Request error for {word} via {proxy_url}: {str(e)}")
+            await asyncio.sleep(1.5)
 
     return "ERROR"