Update scraper.py

scraper.py

@@ -2,108 +2,152 @@ import asyncio
 import random
 import re
 import time
-from curl_cffi.requests import AsyncSession
+import aiohttp
 from logger import get_logger
+from state import state
+from parser import parse_html
 
 log = get_logger()
 
-# Shared global pacing throttle to manage single-IP footprint
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 Version/17.4.1 Safari/605.1.15",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
+]
+
+# Singleton synchronization variables for proxy-free execution
+_global_api_hash = None
+_global_cookies = None
+_hash_fetched_at = 0.0
+_hash_lock = asyncio.Lock()
 _rate_limited_until = 0.0
 
+async def _get_valid_api_context() -> tuple:
+    """
+    Centralized Gateway Handshake: Extracts session initialization context 
+    safely while ensuring worker threads don't collide.
+    """
+    global _global_api_hash, _global_cookies, _hash_fetched_at
+    
+    if _global_api_hash and _global_cookies and (time.time() - _hash_fetched_at < 900):
+        return _global_api_hash, _global_cookies
+
+    async with _hash_lock:
+        now = time.time()
+        if _global_api_hash and _global_cookies and (now - _hash_fetched_at < 900):
+            return _global_api_hash, _global_cookies
+
+        headers = {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+        }
+        
+        try:
+            timeout = aiohttp.ClientTimeout(total=10)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get("https://fragment.com/", headers=headers, allow_redirects=True) as resp:
+                    html = await resp.text()
+                    
+                    match = (
+                        re.search(r'"hash"\s*:\s*"([a-f0-9]+)"', html, re.IGNORECASE) or 
+                        re.search(r'hash\s*=\s*["\']([a-f0-9]+)["\']', html, re.IGNORECASE) or
+                        re.search(r'href="/api\?hash=([a-f0-9]+)"', html, re.IGNORECASE)
+                    )
+                    
+                    if match:
+                        _global_api_hash = match.group(1)
+                        _global_cookies = session.cookie_jar.filter_cookies(resp.url)
+                        _hash_fetched_at = now
+                        log.info(f"🌐 [Handshake Match] Session Context Stabilized: {_global_api_hash}")
+                        return _global_api_hash, _global_cookies
+                    else:
+                        log.error("❌ Token context match fault. Security block may be active.")
+        except Exception as e:
+            log.error(f"Handshake connection break: {str(e)}")
+            
+        return (_global_api_hash if _global_api_hash else "FAILED_TOKEN"), _global_cookies
+
 async def check_fragment(word: str, proxy_url: str = None) -> str:
     """
-    Emulated browser engine using curl_cffi to punch through 
-    datacenter IP challenges proxy-free.
+    High-Throughput Endpoint Scraper incorporating exponential fallback 
+    and adaptive sleep windows for proxy-free environments.
     """
     global _rate_limited_until
     word = word.strip().replace("@", "").lower()
-    url = f"https://fragment.com/username/{word}"
     
-    # 4 Retries with progressive backoff pacing matching your matrix
     for attempt in range(1, 5):
         current_time = time.time()
-        
-        # Enforce strategic delay gaps between single-IP connections
         if current_time < _rate_limited_until:
+            # 2.5s structural padding allows Fragment's server-side leaky bucket pool to clear cleanly
             await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.5, 1.5))
             
+        api_hash, session_cookies = await _get_valid_api_context()
+        if api_hash == "FAILED_TOKEN":
+            await asyncio.sleep(3.0)
+            return "ERROR"
+
         try:
-            # We open an isolated impersonation session to clear sockets cleanly
-            async with AsyncSession(impersonate="chrome120", timeout=12) as session:
-                resp = await session.get(url, allow_redirects=True)
-                
-                # Capture structural rate limits or bans
-                if resp.status_code in [429, 403]:
-                    backoff = 6 + (2 ** attempt) + random.uniform(1.0, 2.5)
-                    _rate_limited_until = time.time() + backoff
-                    log.warning(f"⚠️ IP Pacing Throttle Engaged (HTTP {resp.status_code}) on '{word}'. Cooling...")
-                    continue
-
-                if resp.status_code != 200:
-                    continue
-
-                html = resp.text
-
-                # Check if the page is a security block layout page
-                if "Just a moment..." in html or "cf-browser-verification" in html or "cloudflare" in html.lower():
-                    backoff = 12 + random.uniform(2.0, 5.0)
-                    _rate_limited_until = time.time() + backoff
-                    log.error("❌ Emulated validation gate triggered. Expanding cooldown window...")
-                    continue
-
-                final_url = str(resp.url)
-
-                # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                # ENGINE 1: NATIVE PROFILE DOM SCANNING
-                # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                if "/username/" in final_url:
-                    status_match = re.search(r'class="tm-section-header-status[^"]*"[^>]*>\s*([^<]+?)\s*</span>', html, re.IGNORECASE)
-                    if status_match:
-                        s = status_match.group(1).strip().lower()
-                        if "sold" in s: return "SOLD"
-                        if "taken" in s: return "TAKEN"
-                        if "auction" in s: return "ON_AUCTION"
-                        if "available" in s: return "AVAILABLE"
-                        if "sale" in s or "purchase" in s: return "FOR_SALE"
-                        return s.upper()
-
-                    if 'class="tm-status-taken"' in html: 
-                        return "TAKEN"
-                    if 'class="tm-status-unavail"' in html or 'tm-section-header-status tm-status-unavail' in html:
-                        if "sold for" in html.lower() or "tm-username-usable" in html.lower() or "recently sold" in html.lower():
-                            return "SOLD"
-                        return "UNAVAILABLE"
-                        
-                    return "AVAILABLE"
-
-                # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                # ENGINE 2: REDIRECTED SEARCH SELECTION INTERCEPT
-                # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                else:
-                    clean_html = re.sub(r'\s+', ' ', html)
-                    search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*<', re.IGNORECASE)
-                    match = search_regex.search(clean_html)
+            headers = {
+                "User-Agent": random.choice(USER_AGENTS),
+                "X-Requested-With": "XMLHttpRequest",
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+                "Origin": "https://fragment.com",
+                "Referer": "https://fragment.com/"
+            }
+
+            payload = {
+                "query": word,
+                "type": "usernames",
+                "method": "searchAuctions"
+            }
+
+            timeout = aiohttp.ClientTimeout(total=10)
+            async with aiohttp.ClientSession(timeout=timeout, cookie_jar=aiohttp.DummyCookieJar()) as session:
+                if session_cookies:
+                    session.cookie_jar.update_cookies(session_cookies)
+
+                async with session.post(f"https://fragment.com/api?hash={api_hash}", data=payload, headers=headers) as resp:
                     
-                    if match:
-                        s = match.group(1).strip().lower()
-                        if "taken" in s: return "TAKEN"
-                        if "available" in s: return "AVAILABLE"
-                        if "sold" in s: return "SOLD"
-                        if "auction" in s: return "ON_AUCTION"
-                        if "unavailable" in s: return "UNAVAILABLE"
-                        return s.upper()
-
-                    # Direct structural layout fallback scans
-                    fallback_text = clean_html.lower()
-                    if "on auction" in fallback_text: return "ON_AUCTION"
-                    if "sold" in fallback_text: return "SOLD"
-                    if "unavailable" in fallback_text: return "UNAVAILABLE"
-                    if "taken" in fallback_text or "make an offer" in fallback_text: return "TAKEN"
-
-                    return "UNAVAILABLE"
+                    if resp.status in [429, 403]:
+                        backoff = 5 + (3 ** attempt) + random.uniform(1.0, 2.5)
+                        _rate_limited_until = time.time() + backoff
+                        log.warning(f"⚠️ [HTTP {resp.status}] Server boundary challenge. Cooling pool {backoff:.1f}s...")
+                        continue
+
+                    if resp.status != 200:
+                        continue
+
+                    try:
+                        response_json = await resp.json()
+                    except ValueError:
+                        # Intercept Cloudflare JS Engine text injections
+                        backoff = 10 + random.uniform(2.0, 5.0)
+                        _rate_limited_until = time.time() + backoff
+                        log.error("❌ JavaScript Browser Challenge Intercepted. Pacing main loops...")
+                        continue
+
+                    # ⚡ INTERCEPT INLINE ARRAY RATE LIMIT SHIFTS
+                    if isinstance(response_json, list):
+                        backoff = 6 + (2 ** attempt) + random.uniform(1.5, 3.5)
+                        _rate_limited_until = time.time() + backoff
+                        log.warning(f"🛑 [Telemetry Capture] RateLimit Array Payload hit checking '{word}'. Backing off {backoff:.1f}s...")
+                        continue
+                    
+                    if not response_json.get("ok"):
+                        global _global_api_hash
+                        _global_api_hash = None # Instantly wipe cached registry signature to trigger context rebuild
+                        continue
+
+                    html_fragment = response_json.get("html", "")
+                    
+                    # Offload the structural processing cleanly to your parser module
+                    decision = parse_html(html_fragment, str(resp.url), word)
+                    return decision
 
         except Exception as e:
-            log.error(f"Impersonation channel error for '{word}': {str(e)}")
+            log.error(f"Internal endpoint transaction pipeline failure for '{word}': {str(e)}")
             await asyncio.sleep(1.5 * attempt)
 
-    return "ERROR"
+    return "ERROR"