Update scraper.py

scraper.py

@@ -15,47 +15,38 @@ USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
 ]
 
-# Thread-safe global synchronization boundaries for token balancing
+# Thread-safe global telemetry registers and cookie state stores
 _global_api_hash = None
+_global_cookies = None
 _hash_fetched_at = 0.0
 _hash_lock = asyncio.Lock()
 _rate_limited_until = 0.0
 
-async def _get_valid_api_hash(proxy_url: str = None) -> str:
+async def _get_valid_api_context(proxy_url: str = None) -> tuple:
     """
-    Ensures a singular valid session context hash token is distributed across 
-    all background threads simultaneously, utilizing proxy routing to bypass
-    initial Cloudflare blocks on the server IP range.
+    Synchronizes handshakes across concurrent worker processes, caching security
+    tokens and structural cookie strings synchronously.
     """
-    global _global_api_hash, _hash_fetched_at
+    global _global_api_hash, _global_cookies, _hash_fetched_at
     
     async with _hash_lock:
         now = time.time()
-        # Recycle token cache window for 15 minutes before executing refreshing sequences
-        if _global_api_hash and (now - _hash_fetched_at < 900):
-            return _global_api_hash
-
-        # Force proxy utilization on handshake layer to mask server footprint
-        active_proxy = proxy_url
-        if not active_proxy and state.proxies:
-            active_proxy = random.choice(state.proxies)
+        # Keep context valid in memory for 15 minutes before executing renewal
+        if _global_api_hash and _global_cookies and (now - _hash_fetched_at < 900):
+            return _global_api_hash, _global_cookies
 
         headers = {
             "User-Agent": random.choice(USER_AGENTS),
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Language": "en-US,en;q=0.5",
-            "Cache-Control": "no-cache",
-            "Pragma": "no-cache"
         }
         
         try:
             timeout = aiohttp.ClientTimeout(total=10)
             async with aiohttp.ClientSession(timeout=timeout) as session:
-                # Route the home page request through the active proxy tunnel
-                async with session.get("https://fragment.com/", headers=headers, proxy=active_proxy, allow_redirects=True) as resp:
+                async with session.get("https://fragment.com/", headers=headers, proxy=proxy_url, allow_redirects=True) as resp:
                     html = await resp.text()
                     
-                    # Target matching layout configurations capturing nested App.init configurations
                     match = (
                         re.search(r'"hash"\s*:\s*"([a-f0-9]+)"', html, re.IGNORECASE) or 
                         re.search(r'hash\s*=\s*["\']([a-f0-9]+)["\']', html, re.IGNORECASE) or
@@ -64,34 +55,32 @@ async def _get_valid_api_hash(proxy_url: str = None) -> str:
                     
                     if match:
                         _global_api_hash = match.group(1)
+                        _global_cookies = session.cookie_jar.filter_cookies(resp.url)
                         _hash_fetched_at = now
-                        log.info(f"🌐 Successfully Stabilized Global API Hash Token: {_global_api_hash} (via Proxy)")
-                        return _global_api_hash
+                        log.info(f"🌐 Synchronized Security Tokens. Base Hash: {_global_api_hash}")
+                        return _global_api_hash, _global_cookies
                     else:
-                        log.error("❌ Token extraction fault. DOM format structure layout changed or proxy blocked.")
-                        with open("token_dom_error.log", "w", encoding="utf-8") as f:
-                            f.write(html[:2000])
-                            
+                        log.error("❌ Handshake payload parse failure. Initializing emergency drop context.")
         except Exception as e:
-            log.error(f"Global Handshake Sync Failure: {str(e)}")
+            log.error(f"Handshake connection exception: {str(e)}")
             
-        return _global_api_hash if _global_api_hash else "FAILED_TOKEN"
+        return (_global_api_hash if _global_api_hash else "FAILED_TOKEN"), _global_cookies
 
 async def check_fragment(word: str, proxy_url: str = None) -> str:
     """
-    Connects directly to Fragment's internal AJAX endpoint using XMLHttpRequest signatures,
-    bypassing front-end web layout redirection problems entirely.
+    Direct endpoint query module processing explicit payload variants 
+    and capturing hidden type mutations.
     """
     global _rate_limited_until
     word = word.strip().replace("@", "").lower()
     
-    # 4 Retries matching your advanced optimization matrix
     for attempt in range(1, 5):
         current_time = time.time()
         if current_time < _rate_limited_until:
-            await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.1, 0.4))
+            # 2.5s sweet-spot padding allows Fragment's leaky bucket queue to clear cleanly
+            await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.5, 1.5))
             
-        api_hash = await _get_valid_api_hash(proxy_url)
+        api_hash, session_cookies = await _get_valid_api_context(proxy_url)
         if api_hash == "FAILED_TOKEN":
             return "ERROR"
 
@@ -101,8 +90,7 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
                 "X-Requested-With": "XMLHttpRequest",
                 "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
                 "Origin": "https://fragment.com",
-                "Referer": "https://fragment.com/",
-                "Accept": "application/json, text/javascript, */*; q=0.01"
+                "Referer": "https://fragment.com/"
             }
 
             payload = {
@@ -112,36 +100,53 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
             }
 
             timeout = aiohttp.ClientTimeout(total=12)
-            async with aiohttp.ClientSession(timeout=timeout) as session:
+            # Share cookies explicitly across global sessions to avoid context tracking errors
+            async with aiohttp.ClientSession(timeout=timeout, cookie_jar=aiohttp.DummyCookieJar()) as session:
+                if session_cookies:
+                    session.cookie_jar.update_cookies(session_cookies)
+
                 async with session.post(f"https://fragment.com/api?hash={api_hash}", data=payload, headers=headers, proxy=proxy_url) as resp:
                     
                     if resp.status in [429, 403]:
-                        backoff = 4 + (2 ** attempt) + random.uniform(0.5, 1.5)
+                        backoff = 4 + (2 ** attempt) + random.uniform(1.0, 2.5)
                         _rate_limited_until = time.time() + backoff
-                        log.warning(f"⚠️ Proxy soft-banned on '{word}'. Backing off {backoff:.2f}s...")
+                        log.warning(f"⚠️ Hard HTTP {resp.status} intercept on '{word}'. Pacing pool...")
                         continue
 
                     if resp.status != 200:
+                        await asyncio.sleep(1.0)
+                        continue
+
+                    # ⚡ PRODUCTION USE CASE CRITICAL VARIANT INTERCEPT
+                    try:
+                        response_json = await resp.json()
+                    except ValueError:
+                        # Catch Cloudflare challenge scripts hidden in standard text strings
+                        backoff = 5 + random.uniform(1.0, 3.0)
+                        _rate_limited_until = time.time() + backoff
                         continue
 
-                    response_json = await resp.json()
+                    # Intercept Fragment Array Payload RateLimit variation
+                    if isinstance(response_json, list):
+                        backoff = 6 + (2 ** attempt) + random.uniform(1.5, 3.5)
+                        _rate_limited_until = time.time() + backoff
+                        log.warning(f"🛑 [Telemetry Capture] RateLimit Array Payload engaged checking '{word}'. Backing off {backoff:.2f}s...")
+                        continue
                     
+                    # Intercept internal engine errors or tracking token drops
                     if not response_json.get("ok"):
-                        # Force token reload sequence if current register gets dropped by endpoint engine
                         global _global_api_hash
-                        _global_api_hash = None 
+                        _global_api_hash = None # Clear invalid signature cache register instantly
                         continue
 
                     html_fragment = response_json.get("html", "")
                     if not html_fragment.strip():
                         return "AVAILABLE"
 
-                    # Clean whitespace structure to allow seamless single-line tracking
+                    # Flatten spatial formatting structures to allow linear execution lookups
                     clean_html = re.sub(r'\s+', ' ', html_fragment)
                     
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                    # ENGINE 1: STRUCTURAL TABLE LAYER SCANNING
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+                    # Structural string boundary lookups
                     search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*<', re.IGNORECASE)
                     match = search_regex.search(clean_html)
                     
@@ -154,9 +159,7 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
                         if "sale" in s or "purchase" in s: return "FOR_SALE"
                         return s.upper()
 
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-                    # ENGINE 2: GLOBAL FALLBACK LAYER MATCHING
-                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+                    # System text-layer validation block failsafes
                     fallback_text = clean_html.lower()
                     if "on auction" in fallback_text: return "ON_AUCTION"
                     if "sold" in fallback_text: return "SOLD"
@@ -167,7 +170,7 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
                     return "AVAILABLE"
 
         except Exception as e:
-            log.error(f"Internal API pipeline fault for '{word}' via proxy: {str(e)}")
+            log.error(f"Internal endpoint transaction pipeline failure for '{word}': {str(e)}")
             await asyncio.sleep(1.0 * attempt)
 
     return "ERROR"