Update rss_processor.py

rss_processor.py

@@ -10,6 +10,7 @@ import dateutil.parser
 import hashlib
 import json
 import re
+import requests
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -20,6 +21,7 @@ COLLECTION_NAME = "news_articles"
 HF_API_TOKEN = os.getenv("HF_TOKEN")
 REPO_ID = "broadfield-dev/news-rag-db"
 MAX_ARTICLES_PER_FEED = 1000
+RAW_FEEDS_DIR = "raw_rss_feeds"
 
 def initialize_hf_api():
     if not HF_API_TOKEN:
@@ -45,6 +47,22 @@ def clean_text(text):
     text = ' '.join(text.split())
     return text.strip()
 
+def save_raw_rss_to_file(feed_url, content):
+    """Saves the raw RSS content to a file."""
+    if not os.path.exists(RAW_FEEDS_DIR):
+        os.makedirs(RAW_FEEDS_DIR)
+    
+    # Create a safe filename from the URL
+    filename = re.sub(r'[^a-zA-Z0-9]', '_', feed_url) + ".xml"
+    filepath = os.path.join(RAW_FEEDS_DIR, filename)
+    
+    try:
+        with open(filepath, 'w', encoding='utf-8') as f:
+            f.write(content)
+        logger.info(f"Saved raw RSS from {feed_url} to {filepath}")
+    except Exception as e:
+        logger.error(f"Could not save raw RSS from {feed_url}: {e}")
+
 def fetch_rss_feeds():
     articles = []
     seen_links = set()
@@ -65,7 +83,13 @@ def fetch_rss_feeds():
 
             try:
                 logger.info(f"Fetching {feed_url}")
-                feed = feedparser.parse(feed_url)
+                # Fetch raw content first to save it
+                response = requests.get(feed_url, headers={'User-Agent': 'Mozilla/5.0'})
+                response.raise_for_status()
+                raw_content = response.text
+                save_raw_rss_to_file(feed_url, raw_content)
+
+                feed = feedparser.parse(raw_content)
                 if feed.bozo:
                     logger.warning(f"Parse error for {feed_url}: {feed.bozo_exception}")
                     continue
@@ -118,8 +142,10 @@ def fetch_rss_feeds():
                         "category": category,
                         "image": image,
                     })
+            except requests.exceptions.RequestException as e:
+                logger.error(f"Error fetching {feed_url}: {e}")
             except Exception as e:
-                logger.error(f"Error fetching or parsing {feed_url}: {e}")
+                logger.error(f"Error processing {feed_url}: {e}")
     
     logger.info(f"Total unique articles fetched: {len(articles)}")
     return articles