Update scraper.py

scraper.py

@@ -4,104 +4,138 @@ import re
 import time
 import aiohttp
 from logger import get_logger
+from state import state
 
 log = get_logger()
 
 USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 Version/17.4.1 Safari/605.1.15",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0.0.0 Safari/537.36"
 ]
 
+# Thread-safe global cache for token balancing
+_global_api_hash = None
+_hash_fetched_at = 0.0
+_hash_lock = asyncio.Lock()
+_rate_limited_until = 0.0
+
+async def _get_valid_api_hash(proxy_url: str = None) -> str:
+    """
+    Ensures a singular valid session context hash token is distributed across 
+    all background threads simultaneously, mitigating Cloudflare handshake overhead.
+    """
+    global _global_api_hash, _hash_fetched_at
+    
+    async with _hash_lock:
+        now = time.time()
+        # Recycle token cache window for 15 minutes before executing refreshing sequences
+        if _global_api_hash and (now - _hash_fetched_at < 900):
+            return _global_api_hash
+
+        headers = {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        }
+        
+        try:
+            timeout = aiohttp.ClientTimeout(total=10)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get("https://fragment.com/", headers=headers, proxy=proxy_url, allow_redirects=True) as resp:
+                    html = await resp.text()
+                    
+                    match = re.search(r'href="/api\?hash=([a-f0-9]+)"', html) or re.search(r'hash=([a-f0-9]{16,})', html)
+                    if match:
+                        _global_api_hash = match.group(1)
+                        _hash_fetched_at = now
+                        log.info(f"🌐 Regenerated Global API Session Token: {_global_api_hash}")
+                        return _global_api_hash
+        except Exception as e:
+            log.error(f"Global Handshake Sync Failure: {str(e)}")
+            
+        return _global_api_hash if _global_api_hash else "FAILED_TOKEN"
+
 async def check_fragment(word: str, proxy_url: str = None) -> str:
-    url = f"https://fragment.com/username/{word}"
+    global _rate_limited_until
+    word = word.strip().replace("@", "").lower()
     
-    # 4 Retries matching your advanced optimization matrix
     for attempt in range(1, 5):
+        current_time = time.time()
+        if current_time < _rate_limited_until:
+            await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.1, 0.4))
+            
+        api_hash = await _get_valid_api_hash(proxy_url)
+        if api_hash == "FAILED_TOKEN":
+            return "ERROR"
+
         try:
             headers = {
                 "User-Agent": random.choice(USER_AGENTS),
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                "Accept-Language": "en-US,en;q=0.5",
-                "Referer": "https://fragment.com/"
+                "X-Requested-With": "XMLHttpRequest",
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+                "Origin": "https://fragment.com",
+                "Referer": "https://fragment.com/",
+                "Accept": "application/json, text/javascript, */*; q=0.01"
             }
-            
-            timeout = aiohttp.ClientTimeout(total=10)
+
+            payload = {
+                "query": word,
+                "type": "usernames",
+                "method": "searchAuctions"
+            }
+
+            timeout = aiohttp.ClientTimeout(total=12)
             async with aiohttp.ClientSession(timeout=timeout) as session:
-                
-                # OPTIMIZATION: Start with allow_redirects=False to read metadata headers instantly
-                async with session.get(url, headers=headers, proxy=proxy_url, allow_redirects=False) as resp:
-                    status = resp.status
-                    location = resp.headers.get("Location", "")
+                async with session.post(f"https://fragment.com/api?hash={api_hash}", data=payload, headers=headers, proxy=proxy_url) as resp:
                     
-                    # 🚀 TWO-WAY METADATA REDIRECT DETECTOR (Zero body-parsing overhead)
-                    if status in (301, 302, 303, 307, 308):
-                        location_lower = location.lower()
-                        
-                        # Case A: Explicit Search Page Redirect (Dropped / Unassigned Limbo)
-                        if "?query=" in location_lower or f"query={word.lower()}" in location_lower:
-                            return "UNAVAILABLE"
-                        
-                        # Case B: Firewall Soft-Ban Redirect (Kicked back to clean root index)
-                        elif location == "/" or location.rstrip('/') in ["https://fragment.com", "https://www.fragment.com"]:
-                            backoff = (2 ** attempt) + random.uniform(1.0, 3.0)
-                            log.warning(f"⚠️ Rate limit target hit on '{word}'. Backing off loop thread {backoff:.2f}s...")
-                            await asyncio.sleep(backoff)
-                            continue
-                            
-                    if status in [429, 403]:
-                        backoff = 4 + (2 ** attempt) + random.uniform(1.5, 3.5)
-                        await asyncio.sleep(backoff)
+                    if resp.status in [429, 403]:
+                        backoff = 4 + (2 ** attempt) + random.uniform(0.5, 1.5)
+                        _rate_limited_until = time.time() + backoff
                         continue
-                        
-                    if status == 404:
-                        return "UNAVAILABLE"
-                        
-                    # Standard Profile Resolution Block (No Redirects)
-                    html = await resp.text()
+
+                    if resp.status != 200:
+                        continue
+
+                    response_json = await resp.json()
                     
-                    if "Just a moment..." in html or "cloudflare" in html.lower() or 'cf-browser-verification' in html:
-                        await asyncio.sleep(random.uniform(3.0, 5.0))
+                    if not response_json.get("ok"):
+                        # Force token reload sequence if token data gets invalidated by endpoint engine
+                        global _global_api_hash
+                        _global_api_hash = None 
                         continue
-                        
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                    # ENGINE 1: PROFILE DOM TARGETING
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                    status_match = re.search(r'class="tm-section-header-status[^"]*"[^>]*>\s*([^<]+?)\s*</span>', html, re.IGNORECASE)
-                    if status_match:
-                        s = status_match.group(1).strip().lower()
-                        if "sold" in s: return "SOLD"
-                        if "taken" in s: return "TAKEN"
-                        if "auction" in s: return "ON_AUCTION"
-                        if "available" in s: return "AVAILABLE"
-                        if "sale" in s or "purchase" in s: return "FOR_SALE"
-                        return s.upper()
 
-                    if 'class="tm-status-taken"' in html: return "TAKEN"
-                    if 'class="tm-status-unavail"' in html or 'tm-section-header-status tm-status-unavail' in html:
-                        if "sold for" in html.lower() or "tm-username-usable" in html.lower() or "recently sold" in html.lower():
-                            return "SOLD"
-                        return "UNAVAILABLE"
-                        
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                    # ENGINE 2: FAILSAFE CONVERTED SEARCH LOOKUP LOOP
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                    clean_html = re.sub(r'\s+', ' ', html)
-                    search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*TL', re.IGNORECASE)
+                    html_fragment = response_json.get("html", "")
+                    if not html_fragment.strip():
+                        return "AVAILABLE"
+
+                    # ⚡ PERFORMANCE RE-ENGINEERING: Highly optimized structural lookup layout parsing
+                    clean_html = re.sub(r'\s+', ' ', html_fragment)
+                    
+                    # Target row validation isolating keyword structural bounds
+                    search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*<', re.IGNORECASE)
                     match = search_regex.search(clean_html)
                     
                     if match:
                         s = match.group(1).strip().lower()
-                        if "taken" in s: return "TAKEN"
-                        if "available" in s: return "AVAILABLE"
+                        if "auction" in s or "bidding" in s: return "ON_AUCTION"
                         if "sold" in s: return "SOLD"
-                        if "auction" in s: return "ON_AUCTION"
                         if "unavailable" in s: return "UNAVAILABLE"
+                        if "taken" in s or "offer" in s: return "TAKEN"
+                        if "sale" in s or "purchase" in s: return "FOR_SALE"
                         return s.upper()
 
+                    # Global failsafe text block resolution fallback
+                    fallback_text = clean_html.lower()
+                    if "on auction" in fallback_text: return "ON_AUCTION"
+                    if "sold" in fallback_text: return "SOLD"
+                    if "unavailable" in fallback_text: return "UNAVAILABLE"
+                    if "taken" in fallback_text or "make an offer" in fallback_text: return "TAKEN"
+                    if "for sale" in fallback_text: return "FOR_SALE"
+
+                    return "AVAILABLE"
+
         except Exception as e:
-            log.error(f"Execution fault for '{word}' over proxy: {str(e)}")
-            await asyncio.sleep(1.5 * attempt)
-            
+            log.error(f"Internal API pipeline fault for '{word}' via proxy: {str(e)}")
+            await asyncio.sleep(1.0 * attempt)
+
     return "ERROR"