Update scraper.py

scraper.py

@@ -1,175 +1,124 @@
 import asyncio
 import random
 import re
-import time
 import aiohttp
 from logger import get_logger
-from state import state
 
 log = get_logger()
 
+# Stablized native browser headers to simulate real organic search traffic
 USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 Version/17.4.1 Safari/605.1.15",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0.0.0 Safari/537.36"
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0"
 ]
 
-# Thread-safe global synchronization gates
-_global_api_hash = None
-_global_cookies = None
-_hash_fetched_at = 0.0
-_hash_lock = asyncio.Lock()
+# Shared pacing registry to slow down the single IP footprint
 _rate_limited_until = 0.0
 
-async def _get_valid_api_context(proxy_url: str = None) -> tuple:
+async def check_fragment(word: str, proxy_url: str = None) -> str:
     """
-    Thread-Safe Concurrency Gatekeeper: Prevents simultaneous worker requests from 
-    clashing on the token endpoint, handling cookie contexts seamlessly.
+    Direct Proxy-Free Web Extraction Engine.
+    Queries the raw HTML layout directly to bypass token tracking mechanisms.
     """
-    global _global_api_hash, _global_cookies, _hash_fetched_at
-    
-    # 1. Check if an active worker already safely grabbed a valid token while we were waiting
-    if _global_api_hash and _global_cookies and (time.time() - _hash_fetched_at < 900):
-        return _global_api_hash, _global_cookies
-
-    # 2. Gatekeeper Lock: Forces workers 1-9 to wait until worker 0 finishes its execution
-    async with _hash_lock:
-        # Double-check inside the lock to see if worker 0 just finished writing it
-        now = time.time()
-        if _global_api_hash and _global_cookies and (now - _hash_fetched_at < 900):
-            return _global_api_hash, _global_cookies
-
-        # Allocate active proxy cleanly
-        active_proxy = proxy_url
-        if not active_proxy and state.proxies:
-            active_proxy = random.choice(state.proxies)
-
-        headers = {
-            "User-Agent": random.choice(USER_AGENTS),
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.5",
-        }
-        
-        # Stagger/Jitter handshake layer requests to prevent simultaneous pool blocking
-        await asyncio.sleep(random.uniform(0.5, 2.0))
-        
-        try:
-            timeout = aiohttp.ClientTimeout(total=12)
-            async with aiohttp.ClientSession(timeout=timeout) as session:
-                async with session.get("https://fragment.com/", headers=headers, proxy=active_proxy, allow_redirects=True) as resp:
-                    html = await resp.text()
-                    
-                    match = (
-                        re.search(r'"hash"\s*:\s*"([a-f0-9]+)"', html, re.IGNORECASE) or 
-                        re.search(r'hash\s*=\s*["\']([a-f0-9]+)["\']', html, re.IGNORECASE) or
-                        re.search(r'href="/api\?hash=([a-f0-9]+)"', html, re.IGNORECASE)
-                    )
-                    
-                    if match:
-                        _global_api_hash = match.group(1)
-                        _global_cookies = session.cookie_jar.filter_cookies(resp.url)
-                        _hash_fetched_at = now
-                        log.info(f"🌐 [Gatekeeper Match] Token Stabilized: {_global_api_hash} (via {active_proxy.split('@')[-1] if '@' in active_proxy else active_proxy})")
-                        return _global_api_hash, _global_cookies
-                    else:
-                        log.error("❌ Handshake parse failure. Falling back to previous cache state.")
-        except Exception as e:
-            log.error(f"Handshake network failure: {str(e)}")
-            
-        return (_global_api_hash if _global_api_hash else "FAILED_TOKEN"), _global_cookies
-
-async def check_fragment(word: str, proxy_url: str = None) -> str:
     global _rate_limited_until
     word = word.strip().replace("@", "").lower()
+    url = f"https://fragment.com/username/{word}"
     
+    # 4 Retries with progressive pacing windows
     for attempt in range(1, 5):
-        current_time = time.time()
+        current_time = time.time() if 'time' in globals() else asyncio.get_event_loop().time()
+        
+        # Pacing throttle block to maximize single-IP longevity
         if current_time < _rate_limited_until:
             await asyncio.sleep(_rate_limited_until - current_time + random.uniform(0.5, 1.5))
             
-        api_hash, session_cookies = await _get_valid_api_context(proxy_url)
-        if api_hash == "FAILED_TOKEN":
-            # If token cannot be retrieved, wait a bit before retrying the loop
-            await asyncio.sleep(2.0)
-            return "ERROR"
+        headers = {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Referer": "https://fragment.com/"
+        }
 
         try:
-            headers = {
-                "User-Agent": random.choice(USER_AGENTS),
-                "X-Requested-With": "XMLHttpRequest",
-                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
-                "Origin": "https://fragment.com",
-                "Referer": "https://fragment.com/"
-            }
-
-            payload = {
-                "query": word,
-                "type": "usernames",
-                "method": "searchAuctions"
-            }
-
-            timeout = aiohttp.ClientTimeout(total=12)
-            async with aiohttp.ClientSession(timeout=timeout, cookie_jar=aiohttp.DummyCookieJar()) as session:
-                if session_cookies:
-                    session.cookie_jar.update_cookies(session_cookies)
-
-                async with session.post(f"https://fragment.com/api?hash={api_hash}", data=payload, headers=headers, proxy=proxy_url) as resp:
+            # Short connection timeouts prevent sockets from hanging indefinitely
+            timeout = aiohttp.ClientTimeout(total=8, connect=3)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                # Setting allow_redirects=True allows us to handle the search table fallback smoothly
+                async with session.get(url, headers=headers, allow_redirects=True) as resp:
                     
+                    # Intercept standard firewall response failures
                     if resp.status in [429, 403]:
-                        backoff = 4 + (2 ** attempt) + random.uniform(1.0, 2.5)
-                        _rate_limited_until = time.time() + backoff
+                        backoff = 5 + (3 ** attempt) + random.uniform(1.0, 3.0)
+                        _rate_limited_until = (time.time() if 'time' in globals() else asyncio.get_event_loop().time()) + backoff
+                        log.warning(f"⚠️ Datacenter IP Throttled (HTTP {resp.status}) on '{word}'. Cooling pool for {backoff:.1f}s...")
                         continue
 
                     if resp.status != 200:
                         continue
 
-                    try:
-                        response_json = await resp.json()
-                    except ValueError:
-                        backoff = 5 + random.uniform(1.0, 3.0)
-                        _rate_limited_until = time.time() + backoff
-                        continue
+                    html = await resp.text()
 
-                    if isinstance(response_json, list):
-                        backoff = 6 + (2 ** attempt) + random.uniform(1.5, 3.5)
-                        _rate_limited_until = time.time() + backoff
-                        log.warning(f"🛑 Array RateLimit Intercept. Pacing pool for {backoff:.2f}s...")
-                        continue
-                    
-                    if not response_json.get("ok"):
-                        global _global_api_hash
-                        _global_api_hash = None # Reset register immediately to force a single fresh update
+                    # Detect if Cloudflare has completely locked the server out
+                    if "Just a moment..." in html or "cf-browser-verification" in html or "cloudflare" in html.lower():
+                        backoff = 10 + random.uniform(2.0, 5.0)
+                        _rate_limited_until = (time.time() if 'time' in globals() else asyncio.get_event_loop().time()) + backoff
+                        log.error("❌ Cloudflare Intercept Encountered. Slowing down engine loops...")
                         continue
 
-                    html_fragment = response_json.get("html", "")
-                    if not html_fragment.strip():
+                    final_url = str(resp.url)
+
+                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+                    # ENGINE 1: DIRECT PROFILE RESOLUTION
+                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+                    if "/username/" in final_url:
+                        status_match = re.search(r'class="tm-section-header-status[^"]*"[^>]*>\s*([^<]+?)\s*</span>', html, re.IGNORECASE)
+                        if status_match:
+                            s = status_match.group(1).strip().lower()
+                            if "sold" in s: return "SOLD"
+                            if "taken" in s: return "TAKEN"
+                            if "auction" in s: return "ON_AUCTION"
+                            if "available" in s: return "AVAILABLE"
+                            if "sale" in s or "purchase" in s: return "FOR_SALE"
+                            return s.upper()
+
+                        if 'class="tm-status-taken"' in html: 
+                            return "TAKEN"
+                        if 'class="tm-status-unavail"' in html or 'tm-section-header-status tm-status-unavail' in html:
+                            if "sold for" in html.lower() or "tm-username-usable" in html.lower() or "recently sold" in html.lower():
+                                return "SOLD"
+                            return "UNAVAILABLE"
+                            
                         return "AVAILABLE"
 
-                    clean_html = re.sub(r'\s+', ' ', html_fragment)
-                    
-                    search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*<', re.IGNORECASE)
-                    match = search_regex.search(clean_html)
-                    
-                    if match:
-                        s = match.group(1).strip().lower()
-                        if "auction" in s or "bidding" in s: return "ON_AUCTION"
-                        if "sold" in s: return "SOLD"
-                        if "unavailable" in s: return "UNAVAILABLE"
-                        if "taken" in s or "offer" in s: return "TAKEN"
-                        if "sale" in s or "purchase" in s: return "FOR_SALE"
-                        return s.upper()
-
-                    fallback_text = clean_html.lower()
-                    if "on auction" in fallback_text: return "ON_AUCTION"
-                    if "sold" in fallback_text: return "SOLD"
-                    if "unavailable" in fallback_text: return "UNAVAILABLE"
-                    if "taken" in fallback_text or "make an offer" in fallback_text: return "TAKEN"
-                    if "for sale" in fallback_text: return "FOR_SALE"
-
-                    return "AVAILABLE"
+                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+                    # ENGINE 2: REDIRECTED SEARCH LOUKUP FALLBACK
+                    # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+                    else:
+                        clean_html = re.sub(r'\s+', ' ', html)
+                        search_regex = re.compile(rf'>@{word}<.*?class="[^"]*status[^"]*"[^>]*>\s*([^<]+?)\s*<', re.IGNORECASE)
+                        match = search_regex.search(clean_html)
+                        
+                        if match:
+                            s = match.group(1).strip().lower()
+                            if "taken" in s: return "TAKEN"
+                            if "available" in s: return "AVAILABLE"
+                            if "sold" in s: return "SOLD"
+                            if "auction" in s: return "ON_AUCTION"
+                            if "unavailable" in s: return "UNAVAILABLE"
+                            return s.upper()
+
+                        # Failsafe textual scans if structures are obfuscated
+                        fallback_text = clean_html.lower()
+                        if "on auction" in fallback_text: return "ON_AUCTION"
+                        if "sold" in fallback_text: return "SOLD"
+                        if "unavailable" in fallback_text: return "UNAVAILABLE"
+                        if "taken" in fallback_text or "make an offer" in fallback_text: return "TAKEN"
+
+                        return "UNAVAILABLE"
 
         except Exception as e:
-            log.error(f"Internal API transaction fault for '{word}': {str(e)}")
+            log.error(f"Network processing bottleneck for '{word}': {str(e)}")
             await asyncio.sleep(1.0 * attempt)
 
     return "ERROR"