Fix URL normalization: strip trailing slash from start_url

src/crawler.py

@@ -219,6 +219,10 @@ class Crawler:
         # Reset blocked state
         self.blocked_reason = None
         
+        # Normalize start_url (remove trailing slash) to ensure consistency
+        # This fixes issues where https://amazon.in/ vs https://amazon.in behave differently
+        start_url = start_url.rstrip('/')
+        
         parsed_start = urlparse(start_url)
         start_domain = parsed_start.netloc
         base_domain = start_domain.replace('www.', '') # simplistic base domain