Enhance: Real News Scraper via RSS & removal of mock fallbacks

app/routes/news.py

@@ -43,30 +43,21 @@ async def search_news(
             articles = fetch_google_news(query, location)
             
             if not articles:
-                print("[NEWS] No articles returned from Google, using fallback")
-                raise ValueError("No articles found")
+                print(f"[NEWS] No articles found for query='{query}', location='{location}'")
+                return {
+                    "articles": [],
+                    "count": 0,
+                    "query": query,
+                    "location": location,
+                    "timestamp": datetime.now().isoformat()
+                }
                 
         except Exception as scrape_error:
-            print(f"[NEWS] Scraping failed: {str(scrape_error)}, using fallback data")
-            # Fallback to mock data if scraping fails
-            articles = [
-                {
-                    "title": f"Construction Industry Update - {location}",
-                    "snippet": f"Latest developments in {query} sector show positive trends with new regulations and infrastructure projects.",
-                    "sentiment": "Positive",
-                    "published_date": datetime.now().isoformat(),
-                    "link": "https://example.com/fallback1",
-                    "source": "Construction News"
-                },
-                {
-                    "title": f"{query.title()} Regulations Updated",
-                    "snippet": "New guidelines introduced to streamline processes and improve safety standards in the construction industry.",
-                    "sentiment": "Neutral",
-                    "published_date": datetime.now().isoformat(),
-                    "link": "https://example.com/fallback2",
-                    "source": "Industry Watch"
-                }
-            ]
+            print(f"[NEWS] Search failed: {str(scrape_error)}")
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Failed to fetch real-time news: {str(scrape_error)}"
+            )
         
         # Limit results
         articles = articles[:limit]

app/services/news_service.py

@@ -34,7 +34,8 @@ def clean_google_url(google_url: str) -> str:
 
 def fetch_google_news(query: str, state: str) -> list:
     """
-    Fetch news articles from Google News by scraping search results.
+    Fetch news articles from Google News via RSS feed.
+    This is much more robust than scraping HTML as it's a structured format.
     
     Args:
         query: Search query (e.g., 'construction law')
@@ -44,51 +45,47 @@ def fetch_google_news(query: str, state: str) -> list:
         List of news article dictionaries with title, link, snippet, sentiment, date
     """
     search_query = quote(f"{query} {state}")
-    url = f"https://www.google.com/search?q={search_query}&tbm=nws"
+    # Google News RSS search URL
+    url = f"https://news.google.com/rss/search?q={search_query}&hl=en-IN&gl=IN&ceid=IN:en"
 
     try:
         res = requests.get(url, headers=HEADERS, timeout=10)
         res.raise_for_status()
     except requests.RequestException as e:
-        print(f"❌ Request failed: {e}")
+        print(f"❌ RSS Request failed: {e}")
         return []
 
-    soup = BeautifulSoup(res.text, "html.parser")
+    # Parse RSS XML
+    soup = BeautifulSoup(res.text, "xml")
+    items = soup.find_all("item")
 
     results = []
-    articles = soup.find_all("div", class_="SoaBEf")
-
-    for article in articles:
-        # Extract title
-        title_tag = article.find("div", class_="n0jPhd ynAwRc MBeuO nDgy9d")
-        title = title_tag.text.strip() if title_tag else ""
-
-        # Extract URL and clean it
-        a_tag = article.find("a")
-        link = clean_google_url(a_tag["href"]) if a_tag and a_tag.get("href") else ""
-
-        # Extract snippet
-        snippet_tag = article.find("div", class_="GI74Re nDgy9d")
-        snippet = snippet_tag.text.strip() if snippet_tag else ""
-
-        # Extract date from the correct div
-        date_div = article.find("div", class_="OSrXXb")
-        date_text = ""
-        if date_div:
-            span = date_div.find("span")
-            if span:
-                date_text = span.text.strip()
-
-        # Parse it to datetime
-        parsed_date = dateparser.parse(date_text) if date_text else None
-
-        # Analyze sentiment using BOTH title and snippet for better accuracy
-        # Combine title and snippet as news snippets alone are often neutral
+    
+    for item in items:
+        title = item.title.text if item.title else ""
+        link = item.link.text if item.link else ""
+        
+        # Snippets in RSS are usually just the source, but we can extract a bit more 
+        # or just use the title if snippet is missing. 
+        # RSS usually doesn't have a dedicated snippet field like scraping does,
+        # so we'll use description if available, else blank.
+        description = item.description.text if item.description else ""
+        # Clean up description (it often contains HTML in RSS)
+        snippet = BeautifulSoup(description, "html.parser").get_text().strip() if description else ""
+
+        # Extract source from title usually formatted as "Title - Source"
+        source = "Google News"
+        if " - " in title:
+            source = title.split(" - ")[-1]
+
+        # Extract date
+        pub_date = item.pubDate.text if item.pubDate else ""
+        parsed_date = dateparser.parse(pub_date) if pub_date else None
+
+        # Analyze sentiment using title and snippet
         combined_text = f"{title}. {snippet}"
         sentiment = analyze_sentiment(combined_text)
         
-        print(f"[SENTIMENT] {sentiment}: {title[:50]}...")
-        
         if title and link:
             results.append({
                 "title": title,
@@ -96,13 +93,13 @@ def fetch_google_news(query: str, state: str) -> list:
                 "snippet": snippet,
                 "sentiment": sentiment,  
                 "published_date": parsed_date.isoformat() if parsed_date else None,
-                "source": "Google News"
+                "source": source
             })
-    
+            
     # Sort by date (newest first)
     results.sort(key=lambda x: x["published_date"] or "", reverse=True)
 
-    print(f"✅ Extracted {len(results)} news articles")
+    print(f"✅ Extracted {len(results)} news articles from RSS")
     return results