Update crypto_news_scraper.py

crypto_news_scraper.py

@@ -5,7 +5,7 @@ import re
 
 def scrape_crypto_news(urls: list = None, num_entries: int = 10) -> pd.DataFrame:
     if urls is None:
-        urls = ["https://www.coindesk.com/arc/outboundfeeds/rss/","https://cointelegraph.com/rss","https://cryptopotato.com/feed/","https://decrypt.co/feed"] #"https://cryptonews.com/news/feed/"]
+        urls = ["https://www.coindesk.com/arc/outboundfeeds/rss/","https://cointelegraph.com/rss","https://cryptopotato.com/feed/","https://decrypt.co/feed"]
     if num_entries is None:
         num_entries = 10
     news_dict = {}
@@ -39,17 +39,10 @@ def scrape_crypto_news(urls: list = None, num_entries: int = 10) -> pd.DataFrame
                 plain = re.sub(r"<[^>]+>", " ", html.unescape(raw_html))
                 plain = re.sub(r"\s+", " ", plain).strip()
                 pub = e.get("published", "")
-                news_dict[title] = {"link": link, "published": pub, "description": plain, "source": url}
+                news_dict[title] = {"link": link, "published": pub, "description": plain[:180], "source": url}
 
         except:
             pass
         news_df = pd.DataFrame(news_dict).T
         news_df = news_df.reset_index().rename(columns={"index":"title"})
-        # Add duplicate filtering
-        news_df = news_df.drop_duplicates(subset=['title', 'link'])
-        
-        # Sort by publication date
-        news_df['published'] = pd.to_datetime(news_df['published'], errors='coerce')
-        news_df = news_df.sort_values('published', ascending=False)[['title','published','description']]
-        news_df['description'] = news_df['description'].str.slice(0, 150)  # limit description length
     return news_df