Create scraper.py

scraper.py

@@ -0,0 +1,38 @@
+import asyncio
+import random
+import aiohttp
+from parser import parse_html
+
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/123.0 Safari/537.36",
+]
+
+async def fetch_word(session: aiohttp.ClientSession, word: str) -> str:
+    url = f"https://fragment.com/username/{word}"
+    
+    for _ in range(6): 
+        for attempt in range(3):
+            headers = {"User-Agent": random.choice(USER_AGENTS)}
+            try:
+                async with session.get(url, headers=headers, allow_redirects=True, timeout=15) as resp:
+                    if resp.status == 429:
+                        wait = (2 ** attempt * 3) + random.uniform(1, 3)
+                        await asyncio.sleep(wait)
+                        continue
+                    if resp.status == 200:
+                        html = await resp.text()
+                        status = parse_html(html, str(resp.url), word)
+                        await asyncio.sleep(random.uniform(0.15, 0.6))
+                        
+                        if status != "UNCERTAIN":
+                            return status
+                        else:
+                            break 
+            except Exception:
+                pass
+        
+        await asyncio.sleep(random.uniform(0.8, 2.0))
+        
+    return "UNCERTAIN"