Update rss_processor.py

rss_processor.py

@@ -7,11 +7,13 @@ import logging
 from huggingface_hub import HfApi, login
 import shutil
 import rss_feeds
+
 # Setup logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 # Constants
+MAX_ARTICLES_PER_FEED = 5  # Set to 5 for testing, increase later as needed
 LOCAL_DB_DIR = "chroma_db"
 RSS_FEEDS = rss_feeds.RSS_FEEDS
 
@@ -36,7 +38,10 @@ def fetch_rss_feeds():
             if feed.bozo:
                 logger.warning(f"Parse error for {feed_url}: {feed.bozo_exception}")
                 continue
+            article_count = 0
             for entry in feed.entries:
+                if article_count >= MAX_ARTICLES_PER_FEED:
+                    break
                 title = entry.get("title", "No Title").strip()
                 link = entry.get("link", "").strip()
                 description = entry.get("summary", entry.get("description", "No Description"))
@@ -54,6 +59,7 @@ def fetch_rss_feeds():
                         "category": categorize_feed(feed_url),
                         "image": image,
                     })
+                    article_count += 1
         except Exception as e:
             logger.error(f"Error fetching {feed_url}: {e}")
     logger.info(f"Total articles fetched: {len(articles)}")