Update src/agri_predict/scraper.py

src/agri_predict/scraper.py

@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
 from typing import Optional, List, Dict, Any
 import logging
 from pathlib import Path
+from urllib.parse import urlencode
 
 # Configure logging
 logger = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ class AgmarknetAPIClient:
     INCLUDE_EXCEL = "false"
 
     SCRAPER_API_KEY = "bbbbde6b56c0fde1e2a61c914eb22d14"   # <-- Add your key here
-    SCRAPER_API_URL = "http://api.scraperapi.com"
+    SCRAPER_API_URL = "https://api.scraperapi.com"
 
     TIMEOUT = 30
 
@@ -43,7 +44,7 @@ class AgmarknetAPIClient:
             f"Records: {records_count} | URL: {url}"
         )
 
-    def fetch_data(self, date_str: str):
+    def fetch_market_data(self, date_str: str):
         """Fetch data using ScraperAPI.
 
         Args:
@@ -76,7 +77,7 @@ class AgmarknetAPIClient:
             status_code = response.status_code
 
             data = response.json()
-            records_count = len(data.get("data", [])) if isinstance(data, dict) else 0
+            records_count = self._count_records(data) if isinstance(data, dict) else 0
 
             self._log_api_call(date_str, original_url, status_code, records_count)