Update app.py

app.py

@@ -12,6 +12,7 @@ import plotly.graph_objects as go
 import os
 import json
 from tqdm import tqdm
+import urllib.parse  # IMPORTANT: For joining URLs
 
 # Configuration du logging
 logging.basicConfig(
@@ -154,12 +155,16 @@ class PesticideDataFetcher:
                     #  Don't load details here.  Use the cache.
                     # self._load_substance_details(substance_id) # Removed
 
-            url = data.get("nextLink")
+            next_link = data.get("nextLink")
+            if next_link:
+                # Correctly join the BASE_URL with the relative nextLink
+                url = urllib.parse.urljoin(self.BASE_URL, next_link)
+                params = None  # IMPORTANT: Don't send params again!
+            else:
+                url = None
             logger.info(f"Substances loaded so far: {substances_loaded}")
-            if url:
-                data = self.fetch_data(url)  # No params needed for nextLink
-                if not data:
-                    break
+
+
         if self.use_cache:
             self._save_caches()
 
@@ -246,12 +251,14 @@ class PesticideDataFetcher:
                     products_loaded += 1
                     all_products.append(product)
 
-            url = data.get("nextLink") if isinstance(data, dict) else None
+            next_link = data.get("nextLink") if isinstance(data, dict) else None #Get nextLink
+            if next_link:
+                # Correctly join the BASE_URL with the relative nextLink
+                url = urllib.parse.urljoin(self.BASE_URL, next_link)
+                params = None  # IMPORTANT: Don't send params again!
+            else:
+                url = None
             logger.info(f"Products retrieved so far: {products_loaded}")
-            if url:
-                data = self.fetch_data(url)  # No params needed for nextLink
-                if not data:
-                    break
 
         if self.use_cache:
             self._save_caches()
@@ -279,11 +286,13 @@ class PesticideDataFetcher:
                 logger.error(f"Error retrieving MRLs: {data.get('error', 'No info')}")
                 break
             mrls.extend(data.get("value", []))
-            url = data.get("nextLink")
-            if url:
-                data = self.fetch_data(url)  # No params needed for nextLink
-                if not data:
-                    break
+            next_link = data.get("nextLink")
+            if next_link:
+                url = urllib.parse.urljoin(self.BASE_URL, next_link)
+                params = None
+            else:
+                url = None
+           
 
         self._mrl_cache[product_id] = mrls
         if self.use_cache:
@@ -315,13 +324,15 @@ class PesticideDataFetcher:
                 break
 
             all_mrls.extend(data.get("value", []))
-            url = data.get("nextLink")
-            if url:
-                data = self.fetch_data(url)  # No params needed for nextLink
-                if not data:
-                    break
+            next_link = data.get("nextLink")
+            if next_link:
+                url = urllib.parse.urljoin(self.BASE_URL, next_link)
+                params = None
+            else:
+                url = None
         logger.info(f"MRLs retrieved for substance {substance_id}: {len(all_mrls)}")
         return all_mrls
+    
     def format_date(self, date_str: Optional[str]) -> str:
         """Formats an ISO date string into a readable format."""
         if not date_str: