Spaces:
Sleeping
Sleeping
abdulsalam2121 commited on
Commit ·
701b0a2
1
Parent(s): 5e341a9
Increase max pages and total timeout for improved scraping efficiency
Browse files- app/bot.py +2 -2
- app/listing_scraper.py +2 -2
app/bot.py
CHANGED
|
@@ -100,9 +100,9 @@ class BotRunner:
|
|
| 100 |
session=session,
|
| 101 |
min_price=self.min_price,
|
| 102 |
stop_event=self.stop_event,
|
| 103 |
-
max_pages=
|
| 104 |
page_timeout=30, # 30 seconds per page
|
| 105 |
-
total_timeout=
|
| 106 |
)
|
| 107 |
scraper = ProductScraper(session=session)
|
| 108 |
scraped_count = 0
|
|
|
|
| 100 |
session=session,
|
| 101 |
min_price=self.min_price,
|
| 102 |
stop_event=self.stop_event,
|
| 103 |
+
max_pages=10000, # Allow unlimited pages effectively
|
| 104 |
page_timeout=30, # 30 seconds per page
|
| 105 |
+
total_timeout=3600, # 1 hour total for entire scan
|
| 106 |
)
|
| 107 |
scraper = ProductScraper(session=session)
|
| 108 |
scraped_count = 0
|
app/listing_scraper.py
CHANGED
|
@@ -40,9 +40,9 @@ class ListingScraper:
|
|
| 40 |
max_items: Optional[int] = None,
|
| 41 |
stop_event=None,
|
| 42 |
status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
| 43 |
-
max_pages: int =
|
| 44 |
page_timeout: int = 30,
|
| 45 |
-
total_timeout: int =
|
| 46 |
):
|
| 47 |
self.session = session
|
| 48 |
self.min_price = min_price
|
|
|
|
| 40 |
max_items: Optional[int] = None,
|
| 41 |
stop_event=None,
|
| 42 |
status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
| 43 |
+
max_pages: int = 10000,
|
| 44 |
page_timeout: int = 30,
|
| 45 |
+
total_timeout: int = 3600,
|
| 46 |
):
|
| 47 |
self.session = session
|
| 48 |
self.min_price = min_price
|