Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -88,7 +88,7 @@ def get_random_user_agent(include_searx=False):
|
|
| 88 |
else:
|
| 89 |
return random.choice(user_agents)
|
| 90 |
|
| 91 |
-
@retry(stop=stop_after_attempt(
|
| 92 |
def scrape_with_bs4(url, session):
|
| 93 |
try:
|
| 94 |
headers = {'User-Agent': get_random_user_agent()}
|
|
@@ -120,7 +120,7 @@ def scrape_with_bs4(url, session):
|
|
| 120 |
logger.error(f"Unexpected error scraping {url}: {e}")
|
| 121 |
return {'success': False, 'error': str(e)}
|
| 122 |
|
| 123 |
-
@retry(stop=stop_after_attempt(
|
| 124 |
def scrape_with_trafilatura(url):
|
| 125 |
try:
|
| 126 |
downloaded = fetch_url(url, timeout=10)
|
|
|
|
| 88 |
else:
|
| 89 |
return random.choice(user_agents)
|
| 90 |
|
| 91 |
+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
|
| 92 |
def scrape_with_bs4(url, session):
|
| 93 |
try:
|
| 94 |
headers = {'User-Agent': get_random_user_agent()}
|
|
|
|
| 120 |
logger.error(f"Unexpected error scraping {url}: {e}")
|
| 121 |
return {'success': False, 'error': str(e)}
|
| 122 |
|
| 123 |
+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
|
| 124 |
def scrape_with_trafilatura(url):
|
| 125 |
try:
|
| 126 |
downloaded = fetch_url(url, timeout=10)
|