Muttered3 commited on
Commit
a9e914f
·
verified ·
1 Parent(s): 78f40ef

Update worker.py

Browse files
Files changed (1) hide show
  1. worker.py +0 -8
worker.py CHANGED
@@ -14,25 +14,19 @@ async def start_worker(worker_id: int):
14
  await asyncio.sleep(2)
15
  continue
16
 
17
- # Pop a word from the right side of the queue
18
  word = await r.rpop("frag:queue")
19
  if not word:
20
  await asyncio.sleep(2)
21
  continue
22
 
23
- # Get a random verified proxy
24
  proxy = await db.get_random_proxy()
25
-
26
- # Execute stealth scrape
27
  status = await scraper.check_fragment(word, proxy_url=proxy)
28
 
29
- # If the proxy timed out or crashed, put the word back on the queue and rest
30
  if status == "ERROR":
31
  await r.lpush("frag:queue", word)
32
  await asyncio.sleep(1.5)
33
  continue
34
 
35
- # Success! Log it and continue
36
  if status:
37
  await db.mark_done(word, status)
38
 
@@ -41,12 +35,10 @@ async def start_worker(worker_id: int):
41
  traceback.print_exc()
42
  await asyncio.sleep(2)
43
 
44
- # --- RENAMED TO MATCH YOUR main.py ---
45
  async def run_worker():
46
  print("🚀 Sniper Engine Starting...")
47
  concurrency = await db.get_concurrency()
48
 
49
- # Spawn tasks individually so a single worker crash doesn't kill the pool
50
  tasks = []
51
  for i in range(concurrency):
52
  task = asyncio.create_task(start_worker(i))
 
14
  await asyncio.sleep(2)
15
  continue
16
 
 
17
  word = await r.rpop("frag:queue")
18
  if not word:
19
  await asyncio.sleep(2)
20
  continue
21
 
 
22
  proxy = await db.get_random_proxy()
 
 
23
  status = await scraper.check_fragment(word, proxy_url=proxy)
24
 
 
25
  if status == "ERROR":
26
  await r.lpush("frag:queue", word)
27
  await asyncio.sleep(1.5)
28
  continue
29
 
 
30
  if status:
31
  await db.mark_done(word, status)
32
 
 
35
  traceback.print_exc()
36
  await asyncio.sleep(2)
37
 
 
38
  async def run_worker():
39
  print("🚀 Sniper Engine Starting...")
40
  concurrency = await db.get_concurrency()
41
 
 
42
  tasks = []
43
  for i in range(concurrency):
44
  task = asyncio.create_task(start_worker(i))