Update rss_processor.py

rss_processor.py

@@ -46,7 +46,8 @@ def clean_text(text):
 
 def fetch_rss_feeds():
     articles = []
-    
+    seen_links = set()
+
     try:
         with open(FEEDS_FILE, 'r') as f:
             feed_categories = json.load(f)
@@ -69,8 +70,13 @@ def fetch_rss_feeds():
                     continue
                 
                 for entry in feed.entries[:MAX_ARTICLES_PER_FEED]:
-                    title = entry.get("title", "No Title")
                     link = entry.get("link", "")
+                    if not link or link in seen_links:
+                        continue
+                    
+                    seen_links.add(link)
+
+                    title = entry.get("title", "No Title")
                     description_raw = entry.get("summary", entry.get("description", ""))
                     description = clean_text(description_raw)
 
@@ -114,7 +120,7 @@ def fetch_rss_feeds():
             except Exception as e:
                 logger.error(f"Error fetching or parsing {feed_url}: {e}")
     
-    logger.info(f"Total articles fetched: {len(articles)}")
+    logger.info(f"Total unique articles fetched: {len(articles)}")
     return articles
 
 def process_and_store_articles(articles):