Update worker.py

worker.py

@@ -1,61 +1,57 @@
 import asyncio
-import scraper
 import db
+import scraper
+import traceback
 
-async def start_worker(worker_id):
-    # Reduced printing saves massive CPU cycles
+async def start_worker(worker_id: int):
+    print(f"⚙️ Worker {worker_id} initialized.")
+    r = await db.get_redis()
+    
     while True:
         try:
-            # 1. Check if we are allowed to run
             state = await db.get_state()
             if state.get("running") != "1" or state.get("paused") == "1":
-                await asyncio.sleep(5) # Deep sleep when paused or stopped
+                await asyncio.sleep(2)
                 continue
 
-            # 2. Get next word
-            word = await db.pop_word()
-            
-            # 3. Handle empty queue
+            # Pop a word from the right side of the queue
+            word = await r.rpop("frag:queue")
             if not word:
-                # No words left? Sleep longer to save CPU
-                await asyncio.sleep(10)
+                await asyncio.sleep(2)
                 continue
 
-            # 4. Perform the check using the NEW function name
-            status = await scraper.check_fragment(word)
+            # Get a random verified proxy
+            proxy = await db.get_random_proxy()
+
+            # Execute stealth scrape
+            status = await scraper.check_fragment(word, proxy_url=proxy)
             
+            # If the proxy timed out or crashed, put the word back on the queue and rest
+            if status == "ERROR":
+                await r.lpush("frag:queue", word)
+                await asyncio.sleep(1.5)
+                continue
+
+            # Success! Log it and continue
             if status:
                 await db.mark_done(word, status)
-                # Only print when we find something interesting (saves CPU)
-                if status not in ("TAKEN", "UNAVAILABLE"):
-                    print(f"[HIT] {word} is {status}")
-
-            # 5. Micro-sleep to prevent rate limits
-            await asyncio.sleep(0.5)
 
         except Exception as e:
-            # If database is full or connection drops, sleep 5 seconds
-            err = str(e).lower()
-            if "max number of clients" in err or "too many connections" in err:
-                await asyncio.sleep(5)
-            else:
-                await asyncio.sleep(2)
+            print(f"❌ Worker {worker_id} Error: {str(e)}")
+            traceback.print_exc()
+            await asyncio.sleep(2)
 
-# RENAMED from run_workers() to run_worker() to match main.py
-async def run_worker():
-    print("⚙️ Workers initializing...")
-    while True:
-        try:
-            # Fetch desired speed from DB
-            concurrency = await db.get_concurrency()
-            
-            # Start the requested number of workers
-            tasks = [asyncio.create_task(start_worker(i)) for i in range(concurrency)]
-            
-            # Wait for them to finish (they won't, unless crashed)
-            await asyncio.gather(*tasks)
-        except Exception:
-            await asyncio.sleep(10)
+async def main():
+    print("🚀 Sniper Engine Starting...")
+    concurrency = await db.get_concurrency()
+    
+    # Spawn tasks individually so a single worker crash doesn't kill the pool
+    tasks = []
+    for i in range(concurrency):
+        task = asyncio.create_task(start_worker(i))
+        tasks.append(task)
+        
+    await asyncio.gather(*tasks, return_exceptions=True)
 
 if __name__ == "__main__":
-    asyncio.run(run_worker())
+    asyncio.run(main())