abdulsalam2121 commited on
Commit
701b0a2
·
1 Parent(s): 5e341a9

Increase max pages and total timeout for improved scraping efficiency

Browse files
Files changed (2) hide show
  1. app/bot.py +2 -2
  2. app/listing_scraper.py +2 -2
app/bot.py CHANGED
@@ -100,9 +100,9 @@ class BotRunner:
100
  session=session,
101
  min_price=self.min_price,
102
  stop_event=self.stop_event,
103
- max_pages=100, # Limit to 100 pages per scan
104
  page_timeout=30, # 30 seconds per page
105
- total_timeout=1800, # 30 minutes total for entire scan
106
  )
107
  scraper = ProductScraper(session=session)
108
  scraped_count = 0
 
100
  session=session,
101
  min_price=self.min_price,
102
  stop_event=self.stop_event,
103
+ max_pages=10000, # Allow unlimited pages effectively
104
  page_timeout=30, # 30 seconds per page
105
+ total_timeout=3600, # 1 hour total for entire scan
106
  )
107
  scraper = ProductScraper(session=session)
108
  scraped_count = 0
app/listing_scraper.py CHANGED
@@ -40,9 +40,9 @@ class ListingScraper:
40
  max_items: Optional[int] = None,
41
  stop_event=None,
42
  status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
43
- max_pages: int = 100,
44
  page_timeout: int = 30,
45
- total_timeout: int = 1800,
46
  ):
47
  self.session = session
48
  self.min_price = min_price
 
40
  max_items: Optional[int] = None,
41
  stop_event=None,
42
  status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
43
+ max_pages: int = 10000,
44
  page_timeout: int = 30,
45
+ total_timeout: int = 3600,
46
  ):
47
  self.session = session
48
  self.min_price = min_price