Update scraper.py

scraper.py

@@ -9,6 +9,7 @@ from parser import parse_html
 
 log = get_logger()
 
+# Raw browser fingerprint headers to emulate real user navigation paths
 USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 Version/17.4.1 Safari/605.1.15",
@@ -16,138 +17,73 @@ USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
 ]
 
-# Singleton synchronization variables for proxy-free execution
-_global_api_hash = None
-_global_cookies = None
-_hash_fetched_at = 0.0
-_hash_lock = asyncio.Lock()
+# Shared global pacing timeline to protect single IP reputation windows
 _rate_limited_until = 0.0
 
-async def _get_valid_api_context() -> tuple:
-    """
-    Centralized Gateway Handshake: Extracts session initialization context 
-    safely while ensuring worker threads don't collide.
-    """
-    global _global_api_hash, _global_cookies, _hash_fetched_at
-    
-    if _global_api_hash and _global_cookies and (time.time() - _hash_fetched_at < 900):
-        return _global_api_hash, _global_cookies
-
-    async with _hash_lock:
-        now = time.time()
-        if _global_api_hash and _global_cookies and (now - _hash_fetched_at < 900):
-            return _global_api_hash, _global_cookies
-
-        headers = {
-            "User-Agent": random.choice(USER_AGENTS),
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.5",
-        }
-        
-        try:
-            timeout = aiohttp.ClientTimeout(total=10)
-            async with aiohttp.ClientSession(timeout=timeout) as session:
-                async with session.get("https://fragment.com/", headers=headers, allow_redirects=True) as resp:
-                    html = await resp.text()
-                    
-                    match = (
-                        re.search(r'"hash"\s*:\s*"([a-f0-9]+)"', html, re.IGNORECASE) or 
-                        re.search(r'hash\s*=\s*["\']([a-f0-9]+)["\']', html, re.IGNORECASE) or
-                        re.search(r'href="/api\?hash=([a-f0-9]+)"', html, re.IGNORECASE)
-                    )
-                    
-                    if match:
-                        _global_api_hash = match.group(1)
-                        _global_cookies = session.cookie_jar.filter_cookies(resp.url)
-                        _hash_fetched_at = now
-                        log.info(f"🌐 [Handshake Match] Session Context Stabilized: {_global_api_hash}")
-                        return _global_api_hash, _global_cookies
-                    else:
-                        log.error("❌ Token context match fault. Security block may be active.")
-        except Exception as e:
-            log.error(f"Handshake connection break: {str(e)}")
-            
-        return (_global_api_hash if _global_api_hash else "FAILED_TOKEN"), _global_cookies
-
 async def check_fragment(word: str, proxy_url: str = None) -> str:
     """
-    High-Throughput Endpoint Scraper incorporating exponential fallback 
-    and adaptive sleep windows for proxy-free environments.
+    Direct Public DOM Extraction Engine.
+    Restructured for proxy-free operation to bypass token handshake verification entirely.
     """
     global _rate_limited_until
     word = word.strip().replace("@", "").lower()
+    url = f"https://fragment.com/username/{word}"
     
+    # 4 Retries incorporating adaptive exponential backoff pacing
     for attempt in range(1, 5):
         current_time = time.time()
+        
+        # Enforce rate-limit pacing window dynamically
         if current_time < _rate_limited_until:
-            # 2.5s structural padding allows Fragment's server-side leaky bucket pool to clear cleanly
             await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.5, 1.5))
             
-        api_hash, session_cookies = await _get_valid_api_context()
-        if api_hash == "FAILED_TOKEN":
-            await asyncio.sleep(3.0)
-            return "ERROR"
+        headers = {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Referer": "https://fragment.com/",
+            "Cache-Control": "no-cache",
+            "Pragma": "no-cache"
+        }
 
         try:
-            headers = {
-                "User-Agent": random.choice(USER_AGENTS),
-                "X-Requested-With": "XMLHttpRequest",
-                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
-                "Origin": "https://fragment.com",
-                "Referer": "https://fragment.com/"
-            }
-
-            payload = {
-                "query": word,
-                "type": "usernames",
-                "method": "searchAuctions"
-            }
-
-            timeout = aiohttp.ClientTimeout(total=10)
-            async with aiohttp.ClientSession(timeout=timeout, cookie_jar=aiohttp.DummyCookieJar()) as session:
-                if session_cookies:
-                    session.cookie_jar.update_cookies(session_cookies)
-
-                async with session.post(f"https://fragment.com/api?hash={api_hash}", data=payload, headers=headers) as resp:
+            # Short connect timeout boundaries drop dead sockets cleanly
+            timeout = aiohttp.ClientTimeout(total=12, connect=4)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                
+                # allow_redirects=True is critical to resolve unassigned usernames cleanly
+                async with session.get(url, headers=headers, allow_redirects=True) as resp:
+                    status = resp.status
                     
-                    if resp.status in [429, 403]:
-                        backoff = 5 + (3 ** attempt) + random.uniform(1.0, 2.5)
+                    if status in [429, 403]:
+                        # Back off exponentially to preserve single-IP request pools
+                        backoff = 5 + (3 ** attempt) + random.uniform(1.0, 3.0)
                         _rate_limited_until = time.time() + backoff
-                        log.warning(f"⚠️ [HTTP {resp.status}] Server boundary challenge. Cooling pool {backoff:.1f}s...")
+                        log.warning(f"⚠️ Single IP throttled (HTTP {status}) on '{word}'. Cooling pool for {backoff:.1f}s...")
                         continue
 
-                    if resp.status != 200:
+                    if status != 200:
+                        await asyncio.sleep(1.0)
                         continue
 
-                    try:
-                        response_json = await resp.json()
-                    except ValueError:
-                        # Intercept Cloudflare JS Engine text injections
-                        backoff = 10 + random.uniform(2.0, 5.0)
-                        _rate_limited_until = time.time() + backoff
-                        log.error("❌ JavaScript Browser Challenge Intercepted. Pacing main loops...")
-                        continue
+                    html = await resp.text()
 
-                    # ⚡ INTERCEPT INLINE ARRAY RATE LIMIT SHIFTS
-                    if isinstance(response_json, list):
-                        backoff = 6 + (2 ** attempt) + random.uniform(1.5, 3.5)
+                    # Trap Cloudflare anti-bot challenge scripts directly
+                    if "Just a moment..." in html or "cf-browser-verification" in html or "cloudflare" in html.lower():
+                        backoff = 15 + random.uniform(2.0, 5.0)
                         _rate_limited_until = time.time() + backoff
-                        log.warning(f"🛑 [Telemetry Capture] RateLimit Array Payload hit checking '{word}'. Backing off {backoff:.1f}s...")
-                        continue
-                    
-                    if not response_json.get("ok"):
-                        global _global_api_hash
-                        _global_api_hash = None # Instantly wipe cached registry signature to trigger context rebuild
+                        log.error("❌ Cloudflare Browser Challenge engaged. Slowing worker loops down...")
                         continue
 
-                    html_fragment = response_json.get("html", "")
-                    
-                    # Offload the structural processing cleanly to your parser module
-                    decision = parse_html(html_fragment, str(resp.url), word)
+                    # Capture the resolved destination URL metadata signature
+                    final_url = str(resp.url)
+
+                    # Offload the raw page layout data directly to your processing parser
+                    decision = parse_html(html, final_url, word)
                     return decision
 
         except Exception as e:
-            log.error(f"Internal endpoint transaction pipeline failure for '{word}': {str(e)}")
+            log.error(f"Network pipe transaction failure for '{word}': {str(e)}")
             await asyncio.sleep(1.5 * attempt)
 
-    return "ERROR"
+    return "ERROR"