Update app.py

app.py

@@ -12,7 +12,7 @@ import asyncio
 from curl_cffi.requests import AsyncSession
 from tqdm.asyncio import tqdm
 from fake_headers import Headers
-
+from urllib.parse import urlparse, urlunparse
 
 # Limit the number of concurrent workers
 CONCURRENT_WORKERS = 5
@@ -35,6 +35,21 @@ tokenizer = None
 
 
 
+def get_main_page_url(url):
+
+    try:
+        # Parse the given URL
+        parsed_url = urlparse(url)
+        
+        # Construct the main page URL (scheme + netloc)
+        
+        print(parsed_url.netloc)
+        main_page_url = urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', ''))
+        
+        return main_page_url
+    except Exception as e:
+        return f"Error processing URL: {e}"
+
 
 async def get_page_bs4(url: str, headers):
 
@@ -187,7 +202,9 @@ def classify_website(url):
         FastLanguageModel.for_inference(model)  # Enable native 2x faster inference
 
 
-    urls = [url]
+    main_page_url = get_main_page_url(url)
+    
+    urls = [main_page_url]
     
     final_ans_dict = {}
     print('before scrape_websites')