Muttered3 commited on
Commit
dfd12ec
·
verified ·
1 Parent(s): 1e52cca

Update scraper.py

Browse files
Files changed (1) hide show
  1. scraper.py +7 -15
scraper.py CHANGED
@@ -10,20 +10,14 @@ USER_AGENTS = [
10
  ]
11
 
12
  async def check_fragment(word: str) -> str:
13
- """
14
- Checks Fragment.com using exponential backoff to avoid 429 Rate Limits.
15
- Returns: AVAILABLE, TAKEN, FOR_SALE, ON_AUCTION, SOLD, or UNAVAILABLE
16
- """
17
  url = f"https://fragment.com/username/{word}"
18
 
19
- # Create the session inside the function so bot.py can call it easily
20
  async with aiohttp.ClientSession() as session:
21
  for _ in range(6):
22
  for attempt in range(3):
23
  headers = {"User-Agent": random.choice(USER_AGENTS)}
24
  try:
25
  async with session.get(url, headers=headers, allow_redirects=True, timeout=15) as resp:
26
- # Handle Telegram Rate Limiting (Exponential Backoff)
27
  if resp.status == 429:
28
  wait = (2 ** attempt * 3) + random.uniform(1, 3)
29
  await asyncio.sleep(wait)
@@ -31,9 +25,8 @@ async def check_fragment(word: str) -> str:
31
 
32
  if resp.status == 200:
33
  html = await resp.text()
34
- await asyncio.sleep(random.uniform(0.15, 0.6)) # Human delay
35
 
36
- # Integrated Parser (Replaces external parser_html)
37
  if "tm-status-avail" in html:
38
  return "AVAILABLE"
39
 
@@ -42,24 +35,23 @@ async def check_fragment(word: str) -> str:
42
  return "ON_AUCTION"
43
  if "For sale" in html:
44
  return "FOR_SALE"
 
 
 
45
  return "TAKEN"
46
 
47
  if "tm-status-unavail" in html:
48
  return "UNAVAILABLE"
49
-
50
- if "Sold" in html:
51
- return "SOLD"
52
 
53
- # Fallback checks
54
  if "Available" in html: return "AVAILABLE"
55
  if "Taken" in html: return "TAKEN"
56
 
57
  return "UNAVAILABLE"
58
 
59
  except Exception:
60
- pass # Ignore connection drops and retry
61
 
62
- # Deep sleep before trying the outer loop again
63
  await asyncio.sleep(random.uniform(0.8, 2.0))
64
 
65
- return "UNAVAILABLE"
 
10
  ]
11
 
12
  async def check_fragment(word: str) -> str:
 
 
 
 
13
  url = f"https://fragment.com/username/{word}"
14
 
 
15
  async with aiohttp.ClientSession() as session:
16
  for _ in range(6):
17
  for attempt in range(3):
18
  headers = {"User-Agent": random.choice(USER_AGENTS)}
19
  try:
20
  async with session.get(url, headers=headers, allow_redirects=True, timeout=15) as resp:
 
21
  if resp.status == 429:
22
  wait = (2 ** attempt * 3) + random.uniform(1, 3)
23
  await asyncio.sleep(wait)
 
25
 
26
  if resp.status == 200:
27
  html = await resp.text()
28
+ await asyncio.sleep(random.uniform(0.15, 0.6))
29
 
 
30
  if "tm-status-avail" in html:
31
  return "AVAILABLE"
32
 
 
35
  return "ON_AUCTION"
36
  if "For sale" in html:
37
  return "FOR_SALE"
38
+ # CRITICAL FIX: Looks for the specific sold text, avoiding the nav bar
39
+ if ">Sold for" in html or ">Sold at" in html or 'class="tm-status-text">Sold' in html:
40
+ return "SOLD"
41
  return "TAKEN"
42
 
43
  if "tm-status-unavail" in html:
44
  return "UNAVAILABLE"
 
 
 
45
 
46
+ # Fallbacks
47
  if "Available" in html: return "AVAILABLE"
48
  if "Taken" in html: return "TAKEN"
49
 
50
  return "UNAVAILABLE"
51
 
52
  except Exception:
53
+ pass
54
 
 
55
  await asyncio.sleep(random.uniform(0.8, 2.0))
56
 
57
+ return "UNAVAILABLE"