Update scraper.py

scraper.py

@@ -14,19 +14,11 @@ USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
 ]
 
-# Track adaptive rate limits per proxy instance in memory safely
-_rate_limited_until = 0.0
-
 async def check_fragment(word: str, proxy_url: str = None) -> str:
-    global _rate_limited_until
     url = f"https://fragment.com/username/{word}"
     
     # 4 Retries matching your advanced optimization matrix
     for attempt in range(1, 5):
-        current_time = time.time()
-        if current_time < _rate_limited_until:
-            await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.1, 0.4))
-            
         try:
             headers = {
                 "User-Agent": random.choice(USER_AGENTS),
@@ -53,14 +45,14 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
                         
                         # Case B: Firewall Soft-Ban Redirect (Kicked back to clean root index)
                         elif location == "/" or location.rstrip('/') in ["https://fragment.com", "https://www.fragment.com"]:
-                            backoff = (2 ** attempt) + random.uniform(0.5, 1.5)
-                            _rate_limited_until = time.time() + backoff
-                            log.warning(f"⚠️ Proxy soft-banned on '{word}'. Backing off {backoff:.2f}s...")
+                            backoff = (2 ** attempt) + random.uniform(1.0, 3.0)
+                            log.warning(f"⚠️ Rate limit target hit on '{word}'. Backing off loop thread {backoff:.2f}s...")
+                            await asyncio.sleep(backoff)
                             continue
                             
                     if status in [429, 403]:
-                        backoff = 4 + (2 ** attempt) + random.uniform(0.5, 1.5)
-                        _rate_limited_until = time.time() + backoff
+                        backoff = 4 + (2 ** attempt) + random.uniform(1.5, 3.5)
+                        await asyncio.sleep(backoff)
                         continue
                         
                     if status == 404:
@@ -94,7 +86,6 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
                         
                     # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
                     # ENGINE 2: FAILSAFE CONVERTED SEARCH LOOKUP LOOP
-                    # (Runs if 200 OK was returned but page layout looks structural like raw queries)
                     # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
                     clean_html = re.sub(r'\s+', ' ', html)
                     search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*TL', re.IGNORECASE)
@@ -113,4 +104,4 @@ async def check_fragment(word: str, proxy_url: str = None) -> str:
             log.error(f"Execution fault for '{word}' over proxy: {str(e)}")
             await asyncio.sleep(1.5 * attempt)
             
-    return "ERROR"
+    return "ERROR"