Spaces:
Runtime error
Runtime error
| # -*- coding: utf-8 -*- | |
| """Web Scraping and Preprocessing.ipynb | |
| Automatically generated by Colab. | |
| Original file is located at | |
| https://colab.research.google.com/drive/1AHby4fTck0P3S3AKBi_Mw7rmeRN1fGRi | |
| """ | |
| !pip install requests beautifulsoup4 pandas | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import pandas as pd | |
| YAHOO_URL = "https://finance.yahoo.com/news/" | |
| BUSINESS_INSIDER_URL = "https://www.businessinsider.com/finance" | |
| def scrape_yahoonews(): | |
| response = requests.get(YAHOO_URL, headers={"User-Agent": "Mozilla/5.0"}) | |
| if response.status_code != 200: | |
| print("Failed to fetch news") | |
| return [] | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| news_data = [] | |
| for article in soup.select("a:has(h3)"): | |
| headline_tag = article.find("h3") | |
| if not headline_tag: | |
| continue | |
| headline = headline_tag.text.strip() | |
| link = article["href"] | |
| if not link.startswith("https"): | |
| link = "https://finance.yahoo.com" + link | |
| article_response = requests.get(link, headers={"User-Agent": "Mozilla/5.0"}) | |
| article_soup = BeautifulSoup(article_response.text, "html.parser") | |
| paragraphs = article_soup.find_all("p") | |
| content = " ".join([p.text for p in paragraphs]) | |
| news_data.append({"headline": headline, "content": content, "link": link}) | |
| return news_data | |
| yahoo_news = scrape_yahoonews() | |
| df_yahoo = pd.DataFrame(yahoo_news) | |
| df_yahoo.head() | |
| def clean_yahoo_text(text): | |
| text = text.replace("Oops, something went wrong", "").strip() | |
| return text | |
| df_yahoo["content"] = df_yahoo["content"].apply(clean_yahoo_text) | |
| df_yahoo.to_csv("scraped_yahoo_data.csv", index=False) | |
| def scrape_businessinsider(): | |
| headers = { | |
| "User-Agent": "Mozilla/5.0" | |
| } | |
| response = requests.get(BUSINESS_INSIDER_URL, headers=headers) | |
| if response.status_code != 200: | |
| print("Failed to fetch Business Insider news.") | |
| return [] | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| news_data = [] | |
| for link in soup.find_all("a", href=True): | |
| article_url = link["href"] | |
| title = link.get_text(strip=True) | |
| if not title or "subscribe" in title.lower() or "login" in title.lower() or "/video" in article_url or "/photo" in article_url: | |
| continue | |
| if not article_url.startswith("https"): | |
| article_url = "https://www.businessinsider.com" + article_url | |
| try: | |
| article_response = requests.get(article_url, headers=headers) | |
| article_soup = BeautifulSoup(article_response.text, "html.parser") | |
| paragraphs = article_soup.find_all("p") | |
| content = " ".join([p.get_text(strip=True) for p in paragraphs if len(p.get_text(strip=True)) > 30]) | |
| if content: | |
| news_data.append({"headline": title, "content": content, "link": article_url}) | |
| except Exception as e: | |
| print(f"Error fetching article {article_url}: {e}") | |
| return news_data | |
| insider_news = scrape_businessinsider() | |
| df_insider = pd.DataFrame(insider_news) | |
| df_insider.head() | |
| df_insider.to_csv("scraped_business_insider_news.csv", index=False) | |
| df_combined = pd.concat([df_yahoo, df_insider], ignore_index=True) | |
| df_combined.to_csv("financial_news_combined.csv", index=False) | |
| import pandas as pd | |
| import re | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk.tokenize import word_tokenize | |
| from nltk.stem import WordNetLemmatizer | |
| nltk.download("stopwords") | |
| nltk.download("punkt") | |
| nltk.download("wordnet") | |
| nltk.download("punkt_tab") | |
| df = pd.read_csv("financial_news_combined.csv") | |
| lemmatizer = WordNetLemmatizer() | |
| def text_preprocessing(text): | |
| if pd.isna(text): | |
| return "" | |
| text = text.lower() | |
| text = re.sub(r"http\S+|www\S+", "", text) | |
| text = re.sub(r"[^a-zA-Z\s]", "", text) | |
| words = word_tokenize(text) | |
| words = [lemmatizer.lemmatize(word) for word in words if word not in stopwords.words("english")] | |
| return " ".join(words) | |
| df["preprocessed_content"] = df["content"].astype(str).apply(text_preprocessing) | |
| df.head() | |
| df.to_csv("financial_news_preprocessed.csv", index=False) | |