Create worker.py

worker.py

@@ -0,0 +1,86 @@
+import asyncio
+import time
+import os
+from aiohttp import ClientSession, TCPConnector
+import db
+from scraper import fetch_word
+
+async def run_worker(client):
+    connector = TCPConnector(limit=100)
+    async with ClientSession(connector=connector) as session:
+        while True:
+            try:
+                state = await db.get_state()
+                if state.get("paused") == "1":
+                    await asyncio.sleep(2)
+                    continue
+                if state.get("running") != "1":
+                    await asyncio.sleep(2)
+                    continue
+
+                concurrency = await db.get_concurrency()
+                batch_size = min(concurrency, 50)
+                
+                words = []
+                for _ in range(batch_size):
+                    w = await db.pop_word()
+                    if w: words.append(w)
+                    
+                if not words:
+                    if state.get("running") == "1":
+                        await db.set_state(running="0")
+                        admins = [int(x) for x in os.environ.get("ADMIN_IDS", "").split(",") if x]
+                        for admin in admins:
+                            try:
+                                await client.send_message(admin, "✅ Scan complete")
+                            except Exception:
+                                pass
+                    await asyncio.sleep(5)
+                    continue
+
+                async def process_one(word):
+                    status = await fetch_word(session, word)
+                    await db.mark_done(word, status)
+
+                await asyncio.gather(*[process_one(w) for w in words])
+                
+                state = await db.get_state()
+                processed = int(state.get("processed", 0))
+                
+                if processed > 0 and (processed % 500) < batch_size:
+                    await send_progress(client)
+                    
+            except Exception as e:
+                print(f"[WORKER ERROR] {e}")
+                await asyncio.sleep(5)
+
+async def send_progress(client):
+    try:
+        state = await db.get_state()
+        counts = await db.get_counts()
+        processed = int(state.get("processed", 0))
+        total = int(state.get("total", 0))
+        start_time = float(state.get("start_time", time.time()))
+        
+        pct = (processed / total * 100) if total > 0 else 0
+        elapsed = time.time() - start_time
+        wps = processed / elapsed if elapsed > 0 else 0
+        rem_sec = (total - processed) / wps if wps > 0 else 0
+        
+        h = int(rem_sec // 3600)
+        m = int((rem_sec % 3600) // 60)
+        
+        msg = (
+            f"⚡ Progress: {processed:,}/{total:,} ({pct:.2f}%)\n"
+            f"🔴 Taken: {counts['taken']:,} | 🚫 Unavail: {counts['unavailable']:,}\n"
+            f"⏱ Est. remaining: ~{h}h {m}m"
+        )
+        
+        admins = [int(x) for x in os.environ.get("ADMIN_IDS", "").split(",") if x]
+        for admin in admins:
+            try:
+                await client.send_message(admin, msg)
+            except Exception:
+                pass
+    except Exception as e:
+        print(f"[PROGRESS ERROR] {e}")