Update src/agri_predict/scraper.py

src/agri_predict/scraper.py

@@ -19,89 +19,72 @@ logger.setLevel(logging.INFO)
 
 
 class AgmarknetAPIClient:
-    """Client for Agmarknet API."""
-    
+    """Client for Agmarknet API using ScraperAPI."""
+
     BASE_URL = "https://api.agmarknet.gov.in/v1/prices-and-arrivals/market-report/specific"
     
-    # Fixed parameters
-    COMMODITY_GROUP_ID = 3      # Commodity group for Sesamum
-    COMMODITY_ID = 11           # Sesamum(Sesame,Gingelly,Til)
+    # Fixed Parameters
+    COMMODITY_GROUP_ID = 3
+    COMMODITY_ID = 11
     INCLUDE_EXCEL = "false"
-    
-    # Timeout in seconds
+
+    SCRAPER_API_KEY = "bbbbde6b56c0fde1e2a61c914eb22d14"   # <-- Add your key here
+    SCRAPER_API_URL = "http://api.scraperapi.com"
+
     TIMEOUT = 30
-    
+
     def __init__(self):
-        """Initialize API client."""
         self.session = requests.Session()
-        logger.info("Agmarknet API client initialized")
-    
-    def _log_api_call(self, date_str: str, url: str, status_code: int, 
-                     records_count: int = 0):
-        """Log API call details.
-        
-        Args:
-            date_str: Date string (YYYY-MM-DD)
-            url: Full URL called
-            status_code: HTTP status code
-            records_count: Number of records fetched
-        """
+        logger.info("Agmarknet API client initialized with ScraperAPI")
+
+    def _log_api_call(self, date_str: str, url: str, status_code: int, records_count: int = 0):
         logger.info(
             f"API CALL | Date: {date_str} | Status: {status_code} | "
             f"Records: {records_count} | URL: {url}"
         )
-    
-    def fetch_market_data(self, date: str) -> Optional[Dict[str, Any]]:
-        """Fetch market data for a specific date.
-        
+
+    def fetch_data(self, date_str: str):
+        """Fetch data using ScraperAPI.
+
         Args:
-            date: Date string in format YYYY-MM-DD
-            
+            date_str: Date string (YYYY-MM-DD)
+
         Returns:
-            API response dictionary or None if error
+            JSON response from API
         """
-        url = (
-            f"{self.BASE_URL}?date={date}&"
-            f"commodityGroupId={self.COMMODITY_GROUP_ID}&"
-            f"commodityId={self.COMMODITY_ID}&"
-            f"includeExcel={self.INCLUDE_EXCEL}"
-        )
-        
+        # Original Agmarknet query params
+        query_params = {
+            "commodityGroupId": self.COMMODITY_GROUP_ID,
+            "commodityId": self.COMMODITY_ID,
+            "date": date_str,
+            "includeExcel": self.INCLUDE_EXCEL
+        }
+
+        original_url = f"{self.BASE_URL}?{urlencode(query_params)}"
+
+        # ScraperAPI wrapper URL
+        scraper_params = {
+            "api_key": self.SCRAPER_API_KEY,
+            "url": original_url,
+            "render": "false"
+        }
+
+        scraper_url = f"{self.SCRAPER_API_URL}?{urlencode(scraper_params)}"
+
         try:
-            logger.info(f"Fetching data for date: {date}")
-            response = self.session.get(url, timeout=self.TIMEOUT)
-            response.raise_for_status()
-            
+            response = self.session.get(scraper_url, timeout=self.TIMEOUT)
+            status_code = response.status_code
+
             data = response.json()
-            
-            if data.get("success"):
-                # Count total records
-                total_records = self._count_records(data)
-                self._log_api_call(date, url, response.status_code, total_records)
-                logger.info(
-                    f"✅ Successfully fetched data | Date: {date} | "
-                    f"Total records: {total_records}"
-                )
-                return data
-            else:
-                logger.error(
-                    f"❌ API returned failure | Date: {date} | "
-                    f"Message: {data.get('message', 'Unknown error')}"
-                )
-                return None
-                
-        except requests.exceptions.Timeout:
-            logger.error(f"❌ Timeout error for date: {date}")
-            return None
-        except requests.exceptions.HTTPError as e:
-            logger.error(f"❌ HTTP error for date: {date} | Status: {e.response.status_code}")
-            return None
-        except requests.exceptions.RequestException as e:
-            logger.error(f"❌ Request error for date: {date} | Error: {str(e)}")
-            return None
-        except ValueError as e:
-            logger.error(f"❌ JSON decode error for date: {date} | Error: {str(e)}")
-            return None
+            records_count = len(data.get("data", [])) if isinstance(data, dict) else 0
+
+            self._log_api_call(date_str, original_url, status_code, records_count)
+
+            return data
+
+        except Exception as e:
+            logger.error(f"ScraperAPI request failed for {date_str}: {str(e)}")
+            raise
     
     def fetch_date_range(self, start_date: str, end_date: str) -> List[Dict[str, Any]]:
         """Fetch market data for a date range.