bugs fixed and paid api's fetcing fixed

app/routes/admin.py

@@ -3,6 +3,7 @@ from typing import Dict, List
 import asyncio
 from app.services.news_aggregator import NewsAggregator
 from app.services.cache_service import CacheService
+from app.services.circuit_breaker import get_circuit_breaker
 from app.config import settings
 
 router = APIRouter()
@@ -50,7 +51,11 @@ async def warm_cache():
         - categories_failed: Number of categories that failed
         - details: Per-category results with article counts
     """
-    news_aggregator = NewsAggregator()
+    from app.services.scheduler import _get_shared_aggregator
+    
+    # Fix 3: Using the shared singleton ensures admin cache-warming
+    # respects the same API quotas and circuit breakers as the scheduler.
+    shared_aggregator = _get_shared_aggregator()
     cache_service = CacheService()
     
     results = {
@@ -63,7 +68,7 @@ async def warm_cache():
             print(f"[Cache Warming] Fetching {category}...")
             
             # Fetch articles from news aggregator (tries all providers with failover)
-            articles = await news_aggregator.fetch_by_category(category)
+            articles = await shared_aggregator.fetch_by_category(category)
             
             if articles:
                 # Cache the articles with configured TTL (600 seconds)
@@ -117,8 +122,12 @@ async def get_cache_stats():
         - Cache TTL configuration
         - Provider statistics
     """
+    from app.services.scheduler import _get_shared_aggregator
+    
     cache_service = CacheService()
-    news_aggregator = NewsAggregator()
+    
+    # Fix 3: Get stats from the exact same instance that is doing the fetching
+    shared_aggregator = _get_shared_aggregator()
     
     cached_categories = []
     
@@ -131,7 +140,7 @@ async def get_cache_stats():
             })
     
     # Get provider statistics
-    provider_stats = news_aggregator.get_stats()
+    provider_stats = shared_aggregator.get_stats()
     
     return {
         "cache_ttl": settings.CACHE_TTL,
@@ -239,12 +248,15 @@ async def populate_database():
     - Recovery after database cleanup
     """
     from app.services.appwrite_db import get_appwrite_db
-    from app.services.news_aggregator import NewsAggregator
+    from app.services.scheduler import _get_shared_aggregator
     import asyncio
     
     try:
         appwrite_db = get_appwrite_db()
-        news_aggregator = NewsAggregator()
+        
+        # Fix 3: Use the shared aggregator to respect quotas and circuit breakers
+        # during massive full-database populate operations.
+        shared_aggregator = _get_shared_aggregator()
         
         results = {
             "successful": [],
@@ -256,7 +268,7 @@ async def populate_database():
                 print(f"[DB Populate] Fetching {category}...")
                 
                 # Fetch articles from external APIs
-                articles = await news_aggregator.fetch_by_category(category)
+                articles = await shared_aggregator.fetch_by_category(category)
                 
                 if articles:
                     # Save to Appwrite database
@@ -770,3 +782,60 @@ async def bloom_filter_health_check():
             "recommendation": "Check server logs for detailed error information"
         }
 
+
+# ===========================================
+# Circuit Breaker Management (Fix 1b)
+# ===========================================
+
+@router.post("/circuits/reset")
+async def reset_circuit_breakers():
+    """
+    Emergency Circuit Breaker Reset
+
+    Run this endpoint right after any redeployment where you fixed a broken
+    API key. It does two things:
+
+      1. Deletes all 'circuit:{provider}:state' keys from Upstash Redis,
+         so the newly fixed provider isn't blocked by a stale Redis record.
+
+      2. Resets the in-memory circuit state for all providers back to CLOSED,
+         so the very next scheduler run can immediately try the provider again.
+
+    Without this, a redeployment with a fixed API key could be blocked for
+    up to 1 hour by the old Redis state written by the previous broken key.
+
+    Usage:
+        curl -X POST http://your-server/api/admin/circuits/reset
+    """
+    try:
+        from app.services.upstash_cache import get_upstash_cache
+
+        circuit = get_circuit_breaker()
+
+        # Step 1: Wipe all Redis circuit keys for known providers.
+        cache = get_upstash_cache()
+        known_providers = ["gnews", "newsapi", "newsdata", "google_rss", "medium", "official_cloud"]
+        deleted_keys = []
+
+        for provider in known_providers:
+            key = f"circuit:{provider}:state"
+            result = await cache._execute_command(["DEL", key])
+            if result and int(result) > 0:
+                deleted_keys.append(key)
+
+        # Step 2: Reset the in-memory circuit state (all providers go to CLOSED).
+        circuit.reset()
+
+        return {
+            "success": True,
+            "message": "All circuit breakers have been reset. Paid providers will be tried on the next scheduler run.",
+            "redis_keys_deleted": deleted_keys,
+            "redis_keys_checked": [f"circuit:{p}:state" for p in known_providers],
+            "note": "Run this after fixing a broken API key and redeploying."
+        }
+
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to reset circuit breakers: {str(e)}"
+        )

app/services/appwrite_db.py

@@ -856,7 +856,7 @@ class AppwriteDatabase:
             if text_summary:
                 data['text_summary'] = text_summary
                 
-            self.tablesDB.update_row(
+            await self.tablesDB.update_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=collection_id,
                 document_id=document_id,

app/services/circuit_breaker.py

@@ -9,9 +9,11 @@ Features:
 - Exponential backoff
 - Circuit state: CLOSED → OPEN → HALF_OPEN → CLOSED
 - Per-provider tracking
+- Redis-backed persistence: circuit state survives server restarts
 """
 
 import time
+import asyncio
 import logging
 from typing import Dict, Optional
 from enum import Enum
@@ -30,19 +32,25 @@ class CircuitState(str, Enum):
 class ProviderCircuitBreaker:
     """
     Circuit breaker for news API providers
-    
+
     Prevents repeatedly calling providers that are:
     - Rate limited (HTTP 429)
     - Down (HTTP 5xx)
     - Slow to respond
-    
+
     Strategy:
     - After 3 failures in 5 minutes → OPEN circuit (skip for 1 hour)
     - After 1 hour → HALF_OPEN (allow 1 test request)
     - If test succeeds → CLOSED (normal operation)
     - If test fails → OPEN for another hour
+
+    Redis Persistence (NEW):
+    - When a circuit opens, we write the state to Redis with a 1-hour TTL.
+    - On server boot, we read from Redis to restore any previously open circuits.
+    - When a circuit closes, we delete the Redis key so it doesn't block a recovered provider.
+    - If Redis is unavailable, we fall back gracefully to in-memory only.
     """
-    
+
     def __init__(
         self,
         failure_threshold: int = 3,
@@ -52,7 +60,7 @@ class ProviderCircuitBreaker:
     ):
         """
         Initialize circuit breaker
-        
+
         Args:
             failure_threshold: Number of failures before opening circuit
             failure_window: Time window for counting failures (seconds)
@@ -63,42 +71,127 @@ class ProviderCircuitBreaker:
         self.failure_window = failure_window
         self.open_duration = open_duration
         self.half_open_max_attempts = half_open_max_attempts
-        
+
         # Provider state tracking
         self.states: Dict[str, CircuitState] = defaultdict(lambda: CircuitState.CLOSED)
-        self.failure_counts: Dict[str, int] = defaultdict(int)
-        self.last_failure_time: Dict[str, float] = {}
+        # Fix 1: Track actual timestamps of failures so we can enforce the failure_window.
+        self.failure_timestamps: Dict[str, list[float]] = defaultdict(list)
         self.circuit_open_time: Dict[str, float] = {}
         self.half_open_attempts: Dict[str, int] = defaultdict(int)
-        
+
+        # Known providers — used by the boot-time Redis restore
+        self._known_providers = ["gnews", "newsapi", "newsdata", "google_rss", "medium", "official_cloud"]
+
         logger.info("=" * 70)
         logger.info("⚡ [CIRCUIT BREAKER] Provider protection initialized")
         logger.info(f"   Failure threshold: {failure_threshold} failures")
         logger.info(f"   Failure window: {failure_window}s")
         logger.info(f"   Open duration: {open_duration}s ({open_duration//60} min)")
+        logger.info(f"   Redis persistence: ENABLED")
         logger.info("=" * 70)
-    
+
+        # Kick off a background restore from Redis.
+        # Fix 2: asyncio.get_event_loop() is deprecated.
+        try:
+            loop = asyncio.get_running_loop()
+            loop.create_task(self._load_from_redis())
+        except RuntimeError:
+            # No running event loop (e.g., during synchronous initialization or tests).
+            pass
+
+    # ──────────────────────────────────────────────────────────────────────────
+    # Redis Helpers
+    # ──────────────────────────────────────────────────────────────────────────
+
+    def _redis_key(self, provider: str) -> str:
+        """Build the Redis key for a provider's circuit state."""
+        return f"circuit:{provider}:state"
+
+    async def _load_from_redis(self):
+        """
+        On server boot, check Redis for any circuit states that were open
+        before the server restarted. If we find one, restore it in memory
+        so we don't call a broken API immediately after booting.
+        """
+        try:
+            from app.services.upstash_cache import get_upstash_cache
+            cache = get_upstash_cache()
+
+            for provider in self._known_providers:
+                key = self._redis_key(provider)
+                value = await cache._execute_command(["GET", key])
+
+                if value == "open":
+                    # The circuit was open before the restart — keep it open.
+                    self.states[provider] = CircuitState.OPEN
+                    # We don't know the exact open time, so we set it to "now".
+                    # This means the 1-hour timeout will count from this boot,
+                    # which is safe — the TTL on the Redis key is the real gate.
+                    self.circuit_open_time[provider] = time.time()
+                    logger.info(
+                        "⚡ [CIRCUIT BREAKER] Restored OPEN state for %s from Redis (was open before restart).",
+                        provider
+                    )
+
+        except Exception as e:
+            # Redis is unavailable. That's fine — we start with all circuits CLOSED.
+            logger.debug("[CIRCUIT BREAKER] Redis restore skipped (%s) — starting with clean state.", e)
+
+    async def _persist_open_to_redis(self, provider: str):
+        """
+        Write 'circuit:{provider}:state = open' to Redis with a 1-hour TTL.
+        Called whenever a circuit trips to OPEN.
+        This is fire-and-forget: if Redis is unavailable, we log and move on.
+        """
+        try:
+            from app.services.upstash_cache import get_upstash_cache
+            cache = get_upstash_cache()
+            key = self._redis_key(provider)
+            # SET key "open" EX 3600 — expires in exactly 1 hour, same as open_duration.
+            await cache._execute_command(["SET", key, "open", "EX", self.open_duration])
+            logger.debug("[CIRCUIT BREAKER] Persisted OPEN state for %s to Redis.", provider)
+        except Exception as e:
+            logger.debug("[CIRCUIT BREAKER] Redis write failed for %s (%s) — in-memory state still protects us.", provider, e)
+
+    async def _delete_from_redis(self, provider: str):
+        """
+        Delete 'circuit:{provider}:state' from Redis.
+        Called whenever a circuit recovers to CLOSED, or on a full reset.
+        """
+        try:
+            from app.services.upstash_cache import get_upstash_cache
+            cache = get_upstash_cache()
+            key = self._redis_key(provider)
+            await cache._execute_command(["DEL", key])
+            logger.debug("[CIRCUIT BREAKER] Cleared Redis state for %s.", provider)
+        except Exception as e:
+            logger.debug("[CIRCUIT BREAKER] Redis delete failed for %s (%s) — not a blocker.", provider, e)
+
+    # ──────────────────────────────────────────────────────────────────────────
+    # Core Circuit Breaker Logic
+    # ──────────────────────────────────────────────────────────────────────────
+
     def should_skip(self, provider: str) -> bool:
         """
         Check if provider should be skipped
-        
+
         Args:
             provider: Provider name (e.g., "gnews", "newsapi")
-            
+
         Returns:
             True if provider should be skipped, False otherwise
         """
         current_state = self.states[provider]
         current_time = time.time()
-        
+
         # CLOSED = normal operation, don't skip
         if current_state == CircuitState.CLOSED:
             return False
-        
+
         # OPEN = provider failing, check if should move to HALF_OPEN
         if current_state == CircuitState.OPEN:
             open_time = self.circuit_open_time.get(provider, 0)
-            
+
             # Check if open duration has elapsed
             if current_time - open_time >= self.open_duration:
                 # Move to HALF_OPEN (allow test request)
@@ -111,44 +204,83 @@ class ProviderCircuitBreaker:
                 remaining = int(self.open_duration - (current_time - open_time))
                 logger.debug(f"⚡ Circuit OPEN for {provider} ({remaining}s remaining)")
                 return True
-        
+
         # HALF_OPEN = testing recovery
         if current_state == CircuitState.HALF_OPEN:
             # Allow limited test requests
             if self.half_open_attempts[provider] < self.half_open_max_attempts:
+                # FIX (Bug A): Increment the counter so we don't let infinite
+                # test requests through. The old code checked this counter but
+                # never actually increased it, causing an endless loop.
+                self.half_open_attempts[provider] += 1
                 return False  # Allow test
             else:
-                logger.debug(f"⚡ Circuit HALF_OPEN for {provider} (max tests reached)")
-                return True  # Max tests reached
-        
+                # FIX (Bug B — self-rescue):
+                # The test request went through but no success or failure was
+                # recorded. This happens when the API returned HTTP 200 with
+                # 0 articles — a quiet day with no news, not a broken key.
+                #
+                # If we just return True here and do nothing, the circuit stays
+                # frozen in HALF_OPEN forever because there is no other path out.
+                #
+                # Solution: push it back to OPEN with a fresh 1-hour timer.
+                # The cycle will be:
+                #   OPEN (1 hour) → HALF_OPEN (1 test) → inconclusive → OPEN again
+                # Eventually an actual article response will trigger record_success()
+                # and the circuit will properly close. No permanent freeze.
+                logger.warning(
+                    "⚡ [%s] HALF_OPEN test inconclusive (API reached but returned no articles). "
+                    "Resetting to OPEN for another %d minutes.",
+                    provider, self.open_duration // 60
+                )
+                self.states[provider] = CircuitState.OPEN
+                self.circuit_open_time[provider] = time.time()
+
+                # Persist the new OPEN state to Redis so it survives a restart.
+                try:
+                    loop = asyncio.get_running_loop()
+                    loop.create_task(self._persist_open_to_redis(provider))
+                except RuntimeError:
+                    pass
+
+                return True
+
+
         return False
-    
+
     def record_success(self, provider: str):
         """
         Record successful request
-        
+
         Args:
             provider: Provider name
         """
         current_state = self.states[provider]
-        
-        # Reset failure count
-        self.failure_counts[provider] = 0
-        
+
+        # Reset failure tracking
+        self.failure_timestamps[provider].clear()
+
         # Close circuit if it was open/half-open
         if current_state != CircuitState.CLOSED:
             self.states[provider] = CircuitState.CLOSED
             logger.info(f"✅ Circuit CLOSED for {provider} (recovered)")
-    
+
+            # Clean up the Redis key so this provider isn't blocked after the next restart.
+            try:
+                loop = asyncio.get_running_loop()
+                loop.create_task(self._delete_from_redis(provider))
+            except RuntimeError:
+                pass
+
     def record_failure(
-        self, 
-        provider: str, 
+        self,
+        provider: str,
         error_type: str = "unknown",
         status_code: Optional[int] = None
     ):
         """
         Record failed request
-        
+
         Args:
             provider: Provider name
             error_type: Type of error ("rate_limit", "timeout", "server_error")
@@ -156,90 +288,132 @@ class ProviderCircuitBreaker:
         """
         current_state = self.states[provider]
         current_time = time.time()
-        
-        # Increment failure count
-        self.failure_counts[provider] += 1
-        self.last_failure_time[provider] = current_time
-        
+
+        # Fix 1: Enforce the failure_window by pruning old failures.
+        # Remove any timestamps older than (current_time - self.failure_window)
+        cutoff_time = current_time - self.failure_window
+        self.failure_timestamps[provider] = [
+            ts for ts in self.failure_timestamps[provider] if ts >= cutoff_time
+        ]
+
+        # Append this new failure
+        self.failure_timestamps[provider].append(current_time)
+        current_failure_count = len(self.failure_timestamps[provider])
+
         # Log failure with details
         status_str = f" (HTTP {status_code})" if status_code else ""
         logger.warning(
-            f"⚠️  {provider} failure #{self.failure_counts[provider]}: "
+            f"⚠️  {provider} failure #{current_failure_count} (in last {self.failure_window}s): "
             f"{error_type}{status_str}"
         )
-        
+
         # Check if should open circuit
         if current_state == CircuitState.CLOSED:
             # Check failure window
-            if self.failure_counts[provider] >= self.failure_threshold:
-                # Open circuit
+            if current_failure_count >= self.failure_threshold:
+                # Open circuit in memory first (instant protection)
                 self.states[provider] = CircuitState.OPEN
                 self.circuit_open_time[provider] = current_time
-                
+
                 logger.warning(
                     f"🔴 Circuit OPEN for {provider} "
-                    f"({self.failure_counts[provider]} failures) "
+                    f"({current_failure_count} failures in {self.failure_window}s) "
                     f"- skipping for {self.open_duration//60} minutes"
                 )
-        
+
+                # Persist to Redis so the state survives a server restart.
+                try:
+                    loop = asyncio.get_running_loop()
+                    loop.create_task(self._persist_open_to_redis(provider))
+                except RuntimeError:
+                    pass
+
         # If in HALF_OPEN and fails, go back to OPEN
         elif current_state == CircuitState.HALF_OPEN:
             self.states[provider] = CircuitState.OPEN
             self.circuit_open_time[provider] = current_time
-            
+
             logger.warning(
                 f"🔴 Circuit back to OPEN for {provider} "
                 f"(test failed) - skipping for {self.open_duration//60} minutes"
             )
-    
+
+            # Persist the re-opened state to Redis too.
+            try:
+                loop = asyncio.get_running_loop()
+                loop.create_task(self._persist_open_to_redis(provider))
+            except RuntimeError:
+                pass
+
     def reset(self, provider: Optional[str] = None):
         """
         Reset circuit breaker
-        
+
         Args:
             provider: Provider to reset (None = reset all)
         """
         if provider:
-            # Reset specific provider
+            # Reset specific provider in memory
             self.states[provider] = CircuitState.CLOSED
-            self.failure_counts[provider] = 0
+            self.failure_timestamps[provider].clear()
             self.half_open_attempts[provider] = 0
             logger.info(f"🔄 Circuit reset for {provider}")
+
+            # Also remove the Redis key for this provider
+            try:
+                loop = asyncio.get_running_loop()
+                loop.create_task(self._delete_from_redis(provider))
+            except RuntimeError:
+                pass
         else:
-            # Reset all providers
+            # Reset all providers in memory
             self.states.clear()
-            self.failure_counts.clear()
-            self.last_failure_time.clear()
+            self.failure_timestamps.clear()
             self.circuit_open_time.clear()
             self.half_open_attempts.clear()
             logger.info("🔄 All circuits reset")
-    
+
+            # Remove all Redis keys for known providers
+            try:
+                loop = asyncio.get_running_loop()
+                loop.create_task(self._reset_all_redis_keys())
+            except RuntimeError:
+                pass
+
+    async def _reset_all_redis_keys(self):
+        """Delete all circuit state keys from Redis. Called by reset()."""
+        for provider in self._known_providers:
+            await self._delete_from_redis(provider)
+        logger.info("[CIRCUIT BREAKER] All Redis circuit keys cleared.")
+
     def get_stats(self) -> dict:
         """Get circuit breaker statistics"""
         total_open = sum(1 for s in self.states.values() if s == CircuitState.OPEN)
         total_half_open = sum(1 for s in self.states.values() if s == CircuitState.HALF_OPEN)
         total_closed = sum(1 for s in self.states.values() if s == CircuitState.CLOSED)
-        
+
         # Provider details
         provider_details = {}
         for provider, state in self.states.items():
+            timestamps = self.failure_timestamps.get(provider, [])
+            last_fail = timestamps[-1] if timestamps else None
             provider_details[provider] = {
                 'state': state.value,
-                'failures': self.failure_counts.get(provider, 0),
-                'last_failure': self.last_failure_time.get(provider)
+                'failures': len(timestamps),
+                'last_failure': last_fail
             }
-        
+
         return {
             'total_open': total_open,
             'total_half_open': total_half_open,
             'total_closed': total_closed,
             'providers': provider_details
         }
-    
+
     def print_stats(self):
         """Print circuit breaker statistics"""
         stats = self.get_stats()
-        
+
         logger.info("")
         logger.info("=" * 70)
         logger.info("⚡ [CIRCUIT BREAKER] Provider Status")
@@ -248,20 +422,20 @@ class ProviderCircuitBreaker:
         logger.info(f"   🔹 Half-Open Circuits: {stats['total_half_open']}")
         logger.info(f"   🔹 Closed Circuits: {stats['total_closed']}")
         logger.info("")
-        
+
         for provider, details in stats['providers'].items():
             state_emoji = {
                 'closed': '✅',
                 'open': '🔴',
                 'half_open': '🟡'
             }.get(details['state'], '❓')
-            
+
             logger.info(
                 f"   {state_emoji} {provider.upper()}: "
                 f"{details['state'].upper()} "
                 f"({details['failures']} failures)"
             )
-        
+
         logger.info("=" * 70)
         logger.info("")
 
@@ -273,12 +447,12 @@ _circuit_breaker: Optional[ProviderCircuitBreaker] = None
 def get_circuit_breaker() -> ProviderCircuitBreaker:
     """
     Get or create global circuit breaker instance
-    
+
     Returns:
         ProviderCircuitBreaker: Singleton instance
     """
     global _circuit_breaker
-    
+
     if _circuit_breaker is None:
         _circuit_breaker = ProviderCircuitBreaker(
             failure_threshold=3,  # 3 failures
@@ -286,5 +460,5 @@ def get_circuit_breaker() -> ProviderCircuitBreaker:
             open_duration=3600,   # skip for 1 hour
             half_open_max_attempts=1  # 1 test request
         )
-    
+
     return _circuit_breaker

app/services/news_providers.py

@@ -80,62 +80,70 @@ class GNewsProvider(NewsProvider):
         }
     
     async def fetch_news(self, category: str, limit: int = 20) -> List[Article]:
-        """Fetch news from GNews API"""
+        """
+        Fetch news from GNews API.
+
+        Why no 'from'/'to' date filter here?
+        GNews free/basic tier does NOT support date filtering and returns HTTP 403
+        or an error payload when those params are sent. Removing them lets GNews
+        work reliably on any plan tier. Our data_validation.py freshness gate
+        already rejects old articles downstream, so date filtering still happens
+        — just at the right place.
+        """
         if not self.api_key:
             return []
-        
+
         try:
             query = self.category_map.get(category, category)
             url = f"{self.base_url}/search"
 
-            # Build a window from midnight IST today (converted to UTC) to right now.
-            #
-            # Why IST midnight and not UTC midnight?
-            # IST is UTC+5:30. If we used UTC midnight as the "from" date, GNews
-            # would skip articles published in India between 12:00 AM IST and
-            # 5:30 AM IST — the first 5.5 hours of the Indian day.
-            # By computing IST midnight and converting it to UTC, we tell GNews:
-            # "Give me everything published since the Indian day started".
-            _ist_zone   = ZoneInfo("Asia/Kolkata")
-            _now_ist    = datetime.now(_ist_zone)
-            _cutoff_ist = _now_ist.replace(hour=0, minute=0, second=0, microsecond=0)
-            # Convert IST midnight → UTC so the API gets a valid UTC timestamp.
-            _cutoff_utc = _cutoff_ist.astimezone(timezone.utc)
-            # Current moment in UTC for the "to" bound.
-            _now_utc    = datetime.now(timezone.utc)
-
+            # Simple, plan-compatible request — no date window.
             params = {
                 'q': query,
                 'lang': 'en',
                 'country': 'us',
                 'max': min(limit, 10),  # GNews free tier max 10
                 'apikey': self.api_key,
-                'from': _cutoff_utc.strftime('%Y-%m-%dT%H:%M:%SZ'),  # IST midnight in UTC
-                'to':   _now_utc.strftime('%Y-%m-%dT%H:%M:%SZ'),
             }
-            
+
             async with httpx.AsyncClient(timeout=10.0) as client:
                 response = await client.get(url, params=params)
-                
+
                 if response.status_code == 429:
                     print("[WARN] [GNews] Rate limit hit! Switching to next provider...")
                     self.mark_rate_limited()
                     return []
-                
+
                 if response.status_code == 200:
                     self.request_count += 1
                     data = response.json()
-                    articles = self._parse_response(data, category)
+
+                    # FIX (Bug B): GNews sometimes returns HTTP 200 but puts an
+                    # 'errors' key in the JSON body when the API key is wrong or
+                    # a plan restriction is hit.
+                    # We raise here so the aggregator's except block catches it
+                    # and calls circuit.record_failure() automatically.
+                    # That way the circuit breaker knows this is a real failure,
+                    # not just a quiet day with no news.
+                    if data.get('errors'):
+                        raise RuntimeError(
+                            f"[GNews] API error payload: {data.get('errors')}"
+                        )
+
                     articles = self._parse_response(data, category)
                     if articles:
                         print(f"[SUCCESS] [GNews] Fetched {len(articles)} articles successfully")
                     else:
-                        print("[WARN] [GNews] No articles found in response")
+                        print("[WARN] [GNews] No articles this run (API is healthy, just quiet)")
                     return articles
                 else:
                     print(f"[ERROR] [GNews] HTTP {response.status_code} error")
-                
+
                 return []
+        except RuntimeError:
+            # Re-raise RuntimeError (our intentional error-payload signal)
+            # so the aggregator's except block records this as a circuit failure.
+            raise
         except Exception as e:
             print(f"[ERROR] [GNews] API error: {e}")
             return []
@@ -197,45 +205,63 @@ class NewsAPIProvider(NewsProvider):
         }
     
     async def fetch_news(self, category: str, limit: int = 20) -> List[Article]:
-        """Fetch news from NewsAPI"""
+        """
+        Fetch news from NewsAPI.
+
+        Why no 'from' date filter here?
+        Some NewsAPI plan tiers restrict date filtering and return status='error'
+        when a date param is included. Removing the filter makes the request
+        plan-agnostic. Our data_validation.py freshness gate handles date
+        filtering downstream.
+        """
         if not self.api_key:
             return []
-        
+
         try:
             query = self.category_keywords.get(category, category)
             url = f"{self.base_url}/everything"
 
-            # Ask NewsAPI for articles published since midnight IST today.
-            # We compute IST midnight and convert it to UTC before sending it
-            # to the API, because NewsAPI expects UTC timestamps.
-            # This gives Indian users full coverage from their midnight onwards.
-            _ist_zone   = ZoneInfo("Asia/Kolkata")
-            _now_ist    = datetime.now(_ist_zone)
-            _cutoff_ist = _now_ist.replace(hour=0, minute=0, second=0, microsecond=0)
-            _cutoff_utc = _cutoff_ist.astimezone(timezone.utc)  # Convert to UTC for the API
-
+            # Simple, plan-compatible request — no date window.
             params = {
                 'q': query,
                 'language': 'en',
                 'sortBy': 'publishedAt',
                 'pageSize': min(limit, 20),
                 'apiKey': self.api_key,
-                'from': _cutoff_utc.strftime('%Y-%m-%dT%H:%M:%SZ'),  # IST midnight in UTC
             }
-            
+
             async with httpx.AsyncClient(timeout=10.0) as client:
                 response = await client.get(url, params=params)
-                
+
                 if response.status_code == 429 or response.status_code == 426:
                     self.mark_rate_limited()
                     return []
-                
+
                 if response.status_code == 200:
                     self.request_count += 1
                     data = response.json()
-                    return self._parse_response(data, category)
-                
+
+                    # FIX (Bug B): NewsAPI returns HTTP 200 but sets status='error'
+                    # in the JSON when the API key is invalid or a plan restriction
+                    # is hit. We raise here so the aggregator's except block catches
+                    # it and calls circuit.record_failure() automatically.
+                    if data.get('status') == 'error':
+                        raise RuntimeError(
+                            f"[NewsAPI] API error: {data.get('message', 'unknown error')}"
+                        )
+
+                    articles = self._parse_response(data, category)
+                    if articles:
+                        print(f"[SUCCESS] [NewsAPI] Fetched {len(articles)} articles successfully")
+                    else:
+                        print("[WARN] [NewsAPI] No articles this run (API is healthy, just quiet)")
+                    return articles
+
                 return []
+        except RuntimeError:
+            # Re-raise RuntimeError (our intentional error-payload signal)
+            # so the aggregator's except block records this as a circuit failure.
+            raise
         except Exception as e:
             print(f"NewsAPI error: {e}")
             return []