Update app.py

app.py

@@ -9,6 +9,11 @@ from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
 import urllib.robotparser as robotparser
 
+DEFAULT_HEADERS = {
+    "User-Agent": "smolagents-bot/1.0 (+https://example.com/bot) requests"
+}
+
+
 from Gradio_UI import GradioUI
 
 def allowed_by_robots(url: str, user_agent: str = DEFAULT_HEADERS["User-Agent"]) -> bool: