Update scraper.py

scraper.py

@@ -1,57 +1,55 @@
 import asyncio
 import random
-import aiohttp
-
-USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
-]
+from curl_cffi.requests import AsyncSession
 
 async def check_fragment(word: str) -> str:
     url = f"https://fragment.com/username/{word}"
     
-    async with aiohttp.ClientSession() as session:
-        for _ in range(6): 
-            for attempt in range(3):
-                headers = {"User-Agent": random.choice(USER_AGENTS)}
-                try:
-                    async with session.get(url, headers=headers, allow_redirects=True, timeout=15) as resp:
-                        if resp.status == 429:
-                            wait = (2 ** attempt * 3) + random.uniform(1, 3)
-                            await asyncio.sleep(wait)
-                            continue
-                            
-                        if resp.status == 200:
-                            html = await resp.text()
-                            await asyncio.sleep(random.uniform(0.15, 0.6))
-                            
-                            if "tm-status-avail" in html:
-                                return "AVAILABLE"
-                            
-                            if "tm-status-taken" in html:
-                                if "On auction" in html or "tm-section-subscribe" in html:
-                                    return "ON_AUCTION"
-                                if "For sale" in html:
-                                    return "FOR_SALE"
-                                # CRITICAL FIX: Looks for the specific sold text, avoiding the nav bar
-                                if ">Sold for" in html or ">Sold at" in html or 'class="tm-status-text">Sold' in html:
-                                    return "SOLD"
-                                return "TAKEN"
-                            
-                            if "tm-status-unavail" in html:
-                                return "UNAVAILABLE"
+    # Impersonates a real Chrome browser to bypass Cloudflare
+    async with AsyncSession(impersonate="chrome110") as session:
+        for _ in range(4): 
+            try:
+                resp = await session.get(url, timeout=15)
+                
+                # Wait out rate limits
+                if resp.status_code in [429, 403]:
+                    await asyncio.sleep(random.uniform(2, 5))
+                    continue
+                    
+                if resp.status_code == 200:
+                    html = resp.text
+                    
+                    # Double-check if Cloudflare intercepted the page anyway
+                    if "Just a moment..." in html or "Cloudflare" in html:
+                        await asyncio.sleep(random.uniform(3, 6))
+                        continue
+                        
+                    await asyncio.sleep(random.uniform(0.15, 0.6))
+                    
+                    # --- HTML PARSER ---
+                    if "tm-status-avail" in html: return "AVAILABLE"
+                    
+                    if "tm-status-taken" in html:
+                        if "On auction" in html or "tm-section-subscribe" in html: return "ON_AUCTION"
+                        if "For sale" in html: return "FOR_SALE"
+                        if ">Sold for" in html or ">Sold at" in html or 'class="tm-status-text">Sold' in html: return "SOLD"
+                        return "TAKEN"
+                    
+                    if "tm-status-unavail" in html: return "UNAVAILABLE"
 
-                            # Fallbacks
-                            if "Available" in html: return "AVAILABLE"
-                            if "Taken" in html: return "TAKEN"
-                            
-                            return "UNAVAILABLE"
-                            
-                except Exception:
-                    pass 
+                    # Fallbacks
+                    if "Available" in html: return "AVAILABLE"
+                    if "Taken" in html: return "TAKEN"
+                    
+                    # If Fragment redirects to the search page, the word is usually available to claim
+                    if "/?query=" in str(resp.url):
+                        return "AVAILABLE"
+                    
+                    return "UNAVAILABLE"
+                    
+            except Exception:
+                pass 
             
-            await asyncio.sleep(random.uniform(0.8, 2.0))
+            await asyncio.sleep(random.uniform(1.0, 2.0))
             
         return "UNAVAILABLE"