feat: Add pagination support to get_articles API

- Added offset parameter to get_articles method in appwrite_db.py
- Supports pagination with limit and offset query parameters
- Updated logging to show pagination offset in debug messages
- Enables infinite scroll and better data loading performance

app/main.py

@@ -73,6 +73,7 @@ async def root():
     }
 
 @app.get("/health")
+@app.head("/health")  # ← Added for UptimeRobot compatibility
 async def health_check():
     """
     Enhanced health check endpoint with scheduler status

app/routes/news.py

@@ -10,15 +10,25 @@ cache_service = CacheService()
 appwrite_db = get_appwrite_db()
 
 @router.get("/{category}", response_model=NewsResponse)
-async def get_news_by_category(category: str):
+async def get_news_by_category(
+    category: str,
+    limit: int = 20,  # ← Pagination: items per page
+    page: int = 1     # ← Pagination: page number (1-indexed)
+):
     """
-    Get news articles by category with multi-layer caching (Phase 2)
+    Get news articles by category with multi-layer caching and pagination (Phase 4)
     
     **THE GOLDEN RULE: Users NEVER wait for external APIs**
     - Users only read from database (Appwrite)
     - Background workers populate the database every 15 minutes
     - If database is empty, return empty state (workers will fill it soon)
     
+    **Pagination:**
+    - limit: Number of articles per page (default: 20, max: 100)
+    - page: Page number starting from 1 (default: 1)
+    - Example: page=1, limit=20 returns articles 1-20
+    - Example: page=2, limit=20 returns articles 21-40
+    
     Caching Strategy:
     - L1 Cache: Redis (if available) - 600s TTL, ~5ms response
     - L2 Cache: Appwrite Database - persistent, 10-50ms response  
@@ -39,8 +49,15 @@ async def get_news_by_category(category: str):
     - magazines: Tech Magazines
     """
     try:
+        # Validate and cap pagination parameters
+        limit = min(limit, 100)  # Max 100 items per page
+        page = max(page, 1)  # Minimum page 1
+        offset = (page - 1) * limit  # Calculate offset
+        
         # L1: Check Redis cache (fastest path - ~5ms)
-        cached_data = await cache_service.get(f"news:{category}")
+        # Note: Cache key now includes pagination params
+        cache_key = f"news:{category}:p{page}:l{limit}"
+        cached_data = await cache_service.get(cache_key)
         if cached_data:
             return NewsResponse(
                 success=True,
@@ -52,11 +69,11 @@ async def get_news_by_category(category: str):
             )
         
         # L2: Check Appwrite database (fast persistent storage - ~50ms)
-        db_articles = await appwrite_db.get_articles(category, limit=20)
+        db_articles = await appwrite_db.get_articles(category, limit=limit, offset=offset)
         
         if db_articles:
             # Cache the database results in Redis for next request
-            await cache_service.set(f"news:{category}", db_articles)
+            await cache_service.set(cache_key, db_articles)
             
             return NewsResponse(
                 success=True,
@@ -69,7 +86,6 @@ async def get_news_by_category(category: str):
         
         # Database is empty - return empty state
         # Background workers will populate the database every 15 minutes
-        # User should check back soon or wait for automatic refresh
         return NewsResponse(
             success=True,
             category=category,

app/services/appwrite_db.py

@@ -84,13 +84,14 @@ class AppwriteDatabase:
         """
         return hashlib.sha256(url.encode()).hexdigest()[:16]
     
-    async def get_articles(self, category: str, limit: int = 20) -> List[Dict]:
+    async def get_articles(self, category: str, limit: int = 20, offset: int = 0) -> List[Dict]:
         """
-        Get articles by category from Appwrite database (L2 cache)
+        Get articles by category from Appwrite database (L2 cache) with pagination
         
         Args:
             category: News category (e.g., 'ai', 'data-security')
-            limit: Maximum number of articles to return
+            limit: Maximum number of articles to return (default: 20)
+            offset: Number of articles to skip for pagination (default: 0)
         
         Returns:
             List of article dictionaries, sorted by published_at DESC
@@ -99,14 +100,15 @@ class AppwriteDatabase:
             return []
         
         try:
-            # Query articles by category, sorted by published date
+            # Query articles by category, sorted by published date with pagination
             response = self.databases.list_documents(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_COLLECTION_ID,
                 queries=[
                     Query.equal('category', category),  # SDK v4.x uses string value
                     Query.order_desc('published_at'),
-                    Query.limit(limit)
+                    Query.limit(limit),
+                    Query.offset(offset)  # ← Pagination support
                 ]
             )
             
@@ -129,7 +131,7 @@ class AppwriteDatabase:
                     continue
             
             if articles:
-                print(f"✓ Retrieved {len(articles)} articles for '{category}' from Appwrite (L2 cache)")
+                print(f"✓ Retrieved {len(articles)} articles for '{category}' from Appwrite (offset: {offset})")
             
             return articles
             

app/services/scheduler.py

@@ -45,17 +45,23 @@ async def fetch_all_news():
     Runs every 15 minutes to keep database fresh with latest articles.
     This ensures users always get fast responses from L2 cache (Appwrite).
     """
+    start_time = datetime.now()
+    
     logger.info("═" * 80)
     logger.info("📰 [NEWS FETCHER] Starting news fetch for all categories...")
-    logger.info("🕐 Fetch Time: %s", datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+    logger.info("🕐 Start Time: %s", start_time.strftime('%Y-%m-%d %H:%M:%S'))
     logger.info("═" * 80)
     
     news_aggregator = NewsAggregator()
     appwrite_db = get_appwrite_db()
     cache_service = CacheService()
     
+    # Phase 4: Enhanced tracking for observability
     total_fetched = 0
     total_saved = 0
+    total_duplicates = 0
+    total_errors = 0
+    category_stats = {}  # Track per-category stats
     
     for category in CATEGORIES:
         try:
@@ -70,8 +76,20 @@ async def fetch_all_news():
                 # Save to Appwrite database (L2)
                 logger.info("💾 Saving to Appwrite database...")
                 saved_count = await appwrite_db.save_articles(articles)
+                
+                # Calculate duplicates (fetched - saved = duplicates)
+                duplicates = len(articles) - saved_count
+                
                 total_fetched += len(articles)
                 total_saved += saved_count
+                total_duplicates += duplicates
+                
+                # Store category stats
+                category_stats[category] = {
+                    'fetched': len(articles),
+                    'saved': saved_count,
+                    'duplicates': duplicates
+                }
                 
                 # Update Redis cache (L1) if available
                 try:
@@ -80,21 +98,40 @@ async def fetch_all_news():
                 except Exception as e:
                     logger.debug("⚠️  Redis cache unavailable (not critical): %s", e)
                 
-                logger.info("✅ SUCCESS: %d articles fetched, %d new articles saved", len(articles), saved_count)
+                logger.info("✅ SUCCESS: %d fetched, %d new, %d duplicates", 
+                           len(articles), saved_count, duplicates)
             else:
                 logger.warning("⚠️  WARNING: No articles available from any provider")
+                category_stats[category] = {'fetched': 0, 'saved': 0, 'duplicates': 0}
             
         except Exception as e:
+            total_errors += 1
+            category_stats[category] = {'error': str(e)}
             logger.error("❌ ERROR in %s: %s", category, str(e))
             logger.exception("Full traceback:")
             continue
     
+    # Phase 4: Structured end-of-run report
+    end_time = datetime.now()
+    duration = (end_time - start_time).total_seconds()
+    
     logger.info("")
     logger.info("═" * 80)
-    logger.info("🎉 [NEWS FETCHER] COMPLETED!")
-    logger.info("📊 Total fetched: %d articles", total_fetched)
-    logger.info("💾 Total saved: %d new articles", total_saved)
-    logger.info("🕐 Completion time: %s", datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+    logger.info("🎉 [NEWS FETCHER] RUN COMPLETED")
+    logger.info("═" * 80)
+    logger.info("📊 SUMMARY STATISTICS:")
+    logger.info("   🔹 Total Fetched: %d articles", total_fetched)
+    logger.info("   🔹 Total Saved (New): %d articles", total_saved)
+    logger.info("   🔹 Total Duplicates Skipped: %d articles", total_duplicates)
+    logger.info("   🔹 Total Errors: %d categories", total_errors)
+    logger.info("   🔹 Categories Processed: %d/%d", len(CATEGORIES) - total_errors, len(CATEGORIES))
+    logger.info("   🔹 Deduplication Rate: %.1f%%", (total_duplicates / total_fetched * 100) if total_fetched > 0 else 0)
+    logger.info("")
+    logger.info("⏱️  PERFORMANCE:")
+    logger.info("   🔹 Start: %s", start_time.strftime('%H:%M:%S'))
+    logger.info("   🔹 End: %s", end_time.strftime('%H:%M:%S'))
+    logger.info("   🔹 Duration: %.2f seconds", duration)
+    logger.info("   🔹 Throughput: %.1f articles/second", total_fetched / duration if duration > 0 else 0)
     logger.info("═" * 80)