Update scraper.py

scraper.py

@@ -1,55 +1,73 @@
 import asyncio
 import random
 import re
-import aiofiles
 from curl_cffi.requests import AsyncSession
+from logger import get_logger
+
+log = get_logger()
 
 async def check_fragment(word: str, proxy_url: str = None) -> str:
     url = f"https://fragment.com/username/{word}"
-    
     proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
 
-    async with AsyncSession(impersonate="chrome120", proxies=proxies) as session:
-        for attempt in range(4):
-            try:
-                resp = await session.get(url, timeout=15, allow_redirects=True)
+    try:
+        async with AsyncSession(impersonate="chrome120", proxies=proxies) as session:
+            for attempt in range(3):
+                try:
+                    resp = await session.get(url, timeout=15, allow_redirects=True)
+
+                    if resp.status_code in [429, 403]:
+                        log.warning(f"Rate limited on {word}. Retrying...")
+                        await asyncio.sleep(random.uniform(2.0, 4.0))
+                        continue
 
-                if resp.status_code in [429, 403]:
-                    await asyncio.sleep(random.uniform(1.5, 3.5))
-                    continue
+                    html = resp.text
 
-                html = resp.text
+                    if "Just a moment..." in html or "cloudflare" in html.lower():
+                        await asyncio.sleep(random.uniform(2, 4))
+                        continue
 
-                if "Just a moment..." in html or "cloudflare" in html.lower():
-                    await asyncio.sleep(random.uniform(2, 4))
-                    continue
+                    # 1. CANONICAL CHECK (Soft-404 Banishment)
+                    canonical = re.search(r'<link rel="canonical" href="([^"]+)"', html, re.IGNORECASE)
+                    if canonical and "fragment.com/username/" not in canonical.group(1):
+                        return "UNAVAILABLE"
 
-                canonical = re.search(r'<link rel="canonical" href="([^"]+)"', html, re.IGNORECASE)
-                if canonical and "/username/" not in canonical.group(1):
-                    return "UNAVAILABLE"
+                    # 2. PRECISE STATUS EXTRACTION
+                    status_match = re.search(r'class="tm-section-header-status[^"]*"[^>]*>\s*([^<]+?)\s*</span>', html, re.IGNORECASE)
+                    
+                    if status_match:
+                        s = status_match.group(1).strip().lower()
+                        if "sold" in s: return "SOLD"
+                        if "taken" in s: return "TAKEN"
+                        if "auction" in s: return "ON_AUCTION"
+                        if "available" in s: return "AVAILABLE"
+                        if "sale" in s or "purchase" in s: return "FOR_SALE"
 
-                status_match = re.search(r'class="tm-section-header-status[^"]*"[^>]*>\s*([^<]+?)\s*</span>', html, re.IGNORECASE)
+                        # If it found a status span but we don't recognize the text
+                        clean = s.upper()
+                        log.warning(f"UNKNOWN STATUS TEXT for {word}: '{clean}'")
+                        return clean
 
-                if status_match:
-                    s = status_match.group(1).strip().lower()
-                    if "sold" in s: return "SOLD"
-                    if "taken" in s: return "TAKEN"
-                    if "auction" in s: return "ON_AUCTION"
-                    if "available" in s: return "AVAILABLE"
-                    if "sale" in s or "purchase" in s: return "FOR_SALE"
+                    # 3. FALLBACK HTML PARSING (Fixing the False Unavailables)
+                    if 'class="tm-status-taken"' in html: return "TAKEN"
+                    
+                    # 'tm-status-unavail' is used for BOTH Sold and Unavailable. We must differentiate.
+                    if 'class="tm-status-unavail"' in html:
+                        if ">Sold<" in html or ">sold<" in html.lower() or "recently sold" in html.lower():
+                            return "SOLD"
+                        return "UNAVAILABLE"
 
-                    clean = s.upper()
-                    async with aiofiles.open("other.txt", "a", encoding="utf-8") as f:
-                        await f.write(f"{word} - {clean} (Proxy: {proxy_url})\n")
-                    return clean
+                    # If absolutely no markers exist, it might be a new UI layout.
+                    log.error(f"FAILED TO PARSE UI FOR: {word}. HTML snippet logged.")
+                    with open("failed_parse.log", "a", encoding="utf-8") as f:
+                        f.write(f"\n--- {word} ---\n{html[:1000]}\n")
+                    return "ERROR"
 
-                if "tm-status-taken" in html: return "TAKEN"
-                if "tm-status-unavail" in html:
-                    if ">Sold<" in html or ">sold<" in html.lower():
-                        return "SOLD"
-                    return "UNAVAILABLE"
+                except Exception as e:
+                    log.error(f"Request error for {word} (Attempt {attempt+1}): {str(e)}")
+                    await asyncio.sleep(1.5)
 
-            except Exception:
-                await asyncio.sleep(1)
+    except Exception as e:
+        log.error(f"Session error for {word}: {str(e)}")
 
     return "ERROR"