Update app.py

app.py

@@ -17,9 +17,15 @@ source_url = ""
 def fetch_documentation(url: str) -> str:
     try:
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1'
         }
-        response = requests.get(url, headers=headers, timeout=10)
+        response = requests.get(url, headers=headers, timeout=15, allow_redirects=True)
         response.raise_for_status()
         
         soup = BeautifulSoup(response.content, 'html.parser')
@@ -35,7 +41,15 @@ def fetch_documentation(url: str) -> str:
         
         return text
     except Exception as e:
-        raise Exception(f"Error fetching URL: {str(e)}")
+        error_msg = str(e)
+        if "403" in error_msg or "Forbidden" in error_msg:
+            raise Exception(f"Access denied (403 Forbidden). This website blocks automated requests. Try: 1) Using the site's API if available, 2) A different documentation page, 3) GitHub raw content URLs work well (e.g., https://raw.githubusercontent.com/...)")
+        elif "404" in error_msg:
+            raise Exception(f"Page not found (404). Please check the URL is correct.")
+        elif "timeout" in error_msg.lower():
+            raise Exception(f"Request timeout. The website took too long to respond.")
+        else:
+            raise Exception(f"Error fetching URL: {error_msg}")
 
 def chunk_text(text: str, chunk_size: int = 500, overlap: int = 50) -> List[str]:
     sentences = re.split(r'[.!?]+', text)
@@ -186,9 +200,10 @@ with gr.Blocks(theme=gr.themes.Soft(), title="Documentation RAG System") as demo
     gr.Markdown("### Example URLs to try:")
     gr.Examples(
         examples=[
+            ["https://raw.githubusercontent.com/python/cpython/main/README.rst"],
             ["https://docs.python.org/3/tutorial/introduction.html"],
+            ["https://raw.githubusercontent.com/huggingface/transformers/main/README.md"],
             ["https://pytorch.org/docs/stable/torch.html"],
-            ["https://huggingface.co/docs/transformers/quicktour"],
         ],
         inputs=url_input
     )