Upload 15 files

src/README.md

@@ -0,0 +1,120 @@
+# Indian News Scraper
+
+A collection of web scrapers for various Indian news websites that can extract articles based on specific topics.
+
+## Features
+
+- Scrapes articles from major Indian news sources:
+  - Times of India (TOI)
+  - NDTV
+  - WION
+  - Scroll.in
+- Command-line interface for easy use
+- Multithreaded scraping for fast performance
+- Automatic progress saving to prevent data loss
+- CSV output format for easy analysis
+
+## Requirements
+
+- Python 3.7+
+- Chrome browser
+- ChromeDriver (compatible with your Chrome version)
+
+## Installation
+
+1. Clone this repository:
+   ```bash
+   git clone https://github.com/yourusername/indian-news-scraper.git
+   cd indian-news-scraper
+   ```
+
+2. Install the required dependencies:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+3. Make sure you have Chrome and ChromeDriver installed:
+   - Install Chrome: [https://www.google.com/chrome/](https://www.google.com/chrome/)
+   - Download ChromeDriver: [https://chromedriver.chromium.org/downloads](https://chromedriver.chromium.org/downloads)
+   - Make sure ChromeDriver is in your PATH
+
+## Usage
+
+Run the main script with the desired news source and topic:
+
+```bash
+python run_scraper.py --source toi --topic "Climate Change"
+```
+
+### Available News Sources
+
+- `toi` - Times of India
+- `ndtv` - NDTV
+- `wion` - WION News
+- `scroll` - Scroll.in
+
+### Command Line Options
+
+```
+usage: run_scraper.py [-h] --source {toi,ndtv,wion,scroll} --topic TOPIC [--workers WORKERS] [--interval INTERVAL]
+
+Scrape news articles from Indian news websites
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --source {toi,ndtv,wion,scroll}, -s {toi,ndtv,wion,scroll}
+                        News source to scrape from
+  --topic TOPIC, -t TOPIC
+                        Topic to search for (e.g., "Climate Change", "Politics")
+  --workers WORKERS, -w WORKERS
+                        Number of worker threads (default: 4)
+  --interval INTERVAL, -i INTERVAL
+                        Auto-save interval in seconds (default: 300)
+```
+
+### Examples
+
+Scrape articles about "COVID" from Times of India:
+```bash
+python run_scraper.py --source toi --topic COVID
+```
+
+Scrape articles about "Elections" from NDTV with 8 worker threads:
+```bash
+python run_scraper.py --source ndtv --topic Elections --workers 8
+```
+
+Scrape articles about "Climate Change" from Scroll.in with auto-save every minute:
+```bash
+python run_scraper.py --source scroll --topic "Climate Change" --interval 60
+```
+
+## Output
+
+The scraped articles are saved in CSV format in the `output` directory with filenames in the following format:
+```
+{source}_{topic}articles_{timestamp}_{status}.csv
+```
+
+For example:
+```
+output/toi_COVIDarticles_20250407_121530_final.csv
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add some amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Disclaimer
+
+This tool is meant for research and educational purposes only. Please respect the terms of service of the websites you scrape and use the data responsibly.

src/app.py

@@ -0,0 +1,1050 @@
+import streamlit as st
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+import plotly.express as px
+import plotly.graph_objects as go
+from matplotlib.ticker import MaxNLocator
+import os
+import time
+import json
+import requests
+import spacy
+from tqdm import tqdm
+import warnings
+import pandas as pd
+import pussy
+
+# Suppress warnings for cleaner output
+warnings.filterwarnings('ignore')
+
+# Set page configuration
+st.set_page_config(
+    page_title="Sentiment Analysis of RSS Articles",
+    page_icon="📰",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Custom CSS for styling
+def load_css():
+    st.markdown("""
+    <style>
+        .main-header {
+            font-size: 3rem !important;
+            font-weight: 700 !important;
+            text-align: center !important;
+            padding: 2rem 0 !important;
+        }
+        .sub-header {
+            font-size: 2rem !important;
+            font-weight: 600 !important;
+            padding: 1rem 0 !important;
+        }
+        .newspaper-card {
+            background-color: #f8f9fa;
+            border-radius: 10px;
+            padding: 20px;
+            box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+            text-align: center;
+            margin-bottom: 20px;
+        }
+        .newspaper-title {
+            font-size: 1.5rem;
+            font-weight: 600;
+            margin-bottom: 10px;
+        }
+        .entry-page {
+            display: flex;
+            flex-direction: column;
+            justify-content: center;
+            align-items: center;
+            height: 100vh;
+            position: fixed;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+        }
+        .entry-container {
+            text-align: center;
+            background-color: #f8f9fa;
+            padding: 3rem;
+            border-radius: 20px;
+            box-shadow: 0 6px 12px rgba(0, 0, 0, 0.15);
+            max-width: 800px;
+        }
+        .button-container {
+            margin-top: 2rem;
+        }
+        .footer {
+            text-align: center;
+            padding: 1rem;
+            color: #6c757d;
+            margin-top: 2rem;
+            border-top: 1px solid #dee2e6;
+        }
+    </style>
+    """, unsafe_allow_html=True)
+
+# Constants
+INDIA_GEOJSON_URL = 'https://raw.githubusercontent.com/geohacker/india/master/state/india_state.geojson'
+
+# India GeoJSON loading function
+@st.cache_data
+def load_india_geojson():
+    """Load India GeoJSON data for mapping"""
+    try:
+        response = requests.get(INDIA_GEOJSON_URL)
+        return json.loads(response.text)
+    except Exception as e:
+        st.error(f"Failed to load GeoJSON: {e}")
+        st.info("Trying fallback method...")
+        try:
+            # Fallback: pip install geopandas
+            import geopandas as gpd
+            india = gpd.read_file(INDIA_GEOJSON_URL)
+            return json.loads(india.to_json())
+        except:
+            st.error("Error: Could not load India GeoJSON. Please ensure internet connection.")
+            return None
+
+# Load spaCy model (with caching)
+@st.cache_resource
+def load_spacy_model():
+    try:
+        return spacy.load("en_core_web_sm")
+    except OSError:
+        st.info("Downloading spaCy model... This may take a moment.")
+        import subprocess
+        subprocess.call(["python", "-m", "spacy", "download", "en_core_web_sm"])
+        return spacy.load("en_core_web_sm")
+
+# State mapping dictionary
+def get_state_mapping():
+    return {
+        # Standard state names
+        'andhra pradesh': 'Andhra Pradesh',
+        'arunachal pradesh': 'Arunachal Pradesh',
+        'assam': 'Assam',
+        'bihar': 'Bihar',
+        'chhattisgarh': 'Chhattisgarh',
+        'goa': 'Goa',
+        'gujarat': 'Gujarat',
+        'haryana': 'Haryana',
+        'himachal pradesh': 'Himachal Pradesh',
+        'jharkhand': 'Jharkhand',
+        'karnataka': 'Karnataka',
+        'kerala': 'Kerala',
+        'madhya pradesh': 'Madhya Pradesh',
+        'maharashtra': 'Maharashtra',
+        'manipur': 'Manipur',
+        'meghalaya': 'Meghalaya',
+        'mizoram': 'Mizoram',
+        'nagaland': 'Nagaland',
+        'odisha': 'Odisha',
+        'punjab': 'Punjab',
+        'rajasthan': 'Rajasthan',
+        'sikkim': 'Sikkim',
+        'tamil nadu': 'Tamil Nadu',
+        'telangana': 'Telangana',
+        'tripura': 'Tripura',
+        'uttar pradesh': 'Uttar Pradesh',
+        'uttarakhand': 'Uttarakhand',
+        'west bengal': 'West Bengal',
+        # Union Territories
+        'delhi': 'Delhi',
+        'new delhi': 'Delhi',
+        'jammu and kashmir': 'Jammu and Kashmir',
+        'j&k': 'Jammu and Kashmir',
+        'ladakh': 'Ladakh',
+        'chandigarh': 'Chandigarh',
+        'puducherry': 'Puducherry',
+        'pondicherry': 'Puducherry',
+        'andaman and nicobar': 'Andaman and Nicobar Islands',
+        'dadra and nagar haveli': 'Dadra and Nagar Haveli and Daman and Diu',
+        'daman and diu': 'Dadra and Nagar Haveli and Daman and Diu',
+        'lakshadweep': 'Lakshadweep',
+        # Major cities mapped to their states
+        'mumbai': 'Maharashtra',
+        'kolkata': 'West Bengal',
+        'chennai': 'Tamil Nadu',
+        'bangalore': 'Karnataka',
+        'bengaluru': 'Karnataka',
+        'hyderabad': 'Telangana',
+        'ahmedabad': 'Gujarat',
+        'lucknow': 'Uttar Pradesh',
+        'jaipur': 'Rajasthan',
+        'srinagar': 'Jammu and Kashmir',
+        'varanasi': 'Uttar Pradesh',
+        'kochi': 'Kerala',
+        'pune': 'Maharashtra',
+        'agra': 'Uttar Pradesh',
+        'bhopal': 'Madhya Pradesh',
+        'patna': 'Bihar',
+    }
+
+# Function to extract locations from descriptions
+@st.cache_data
+def extract_locations_from_descriptions(df, description_column='desc'):
+    """
+    Extract state names from description column using spaCy
+    """
+    with st.spinner("Extracting location data from articles..."):
+        # Load spaCy model
+        nlp = load_spacy_model()
+        
+        # Get state mapping dictionary
+        state_mapping = get_state_mapping()
+        
+        # Process descriptions to extract locations
+        locations = []
+        
+        # Use a progress bar if processing a large dataset
+        progress_text = "Extracting locations..."
+        progress_bar = st.progress(0)
+        
+        for idx, row in enumerate(df.iterrows()):
+            # Update progress every 100 rows
+            if idx % 100 == 0:
+                progress_bar.progress(min(idx / len(df), 1.0))
+            
+            row = row[1]  # Get the actual row data (second element of the tuple)
+            
+            if pd.isna(row[description_column]):
+                locations.append(None)
+                continue
+                
+            description = str(row[description_column]).lower()
+            doc = nlp(description)
+            
+            # Extract location entities
+            found_locations = []
+            for ent in doc.ents:
+                if ent.label_ in ["GPE", "LOC"]:
+                    loc_name = ent.text.lower()
+                    if loc_name in state_mapping:
+                        found_locations.append(state_mapping[loc_name])
+            
+            # Direct string matching for state names
+            for state_var, standard_name in state_mapping.items():
+                if state_var in description and standard_name not in found_locations:
+                    found_locations.append(standard_name)
+            
+            # Store the first found location, or None if none found
+            locations.append(found_locations[0] if found_locations else None)
+        
+        # Complete progress
+        progress_bar.progress(1.0)
+        
+        # Add locations to dataframe
+        df = df.copy()  # Create a copy to avoid modifying the original
+        df['extracted_location'] = locations
+        
+        st.success(f"Locations extracted. Found locations in {df['extracted_location'].notna().sum()} of {len(df)} articles.")
+        return df
+
+# Function to analyze sentiment by state
+def analyze_sentiment_by_state(df, sentiment_column='sentiment_score'):
+    """
+    Analyze sentiment by state and prepare data for visualization
+    """
+    # Filter to only rows with extracted locations and valid sentiment
+    df_with_locations = df.dropna(subset=['extracted_location', sentiment_column])
+    
+    if len(df_with_locations) == 0:
+        st.warning("No locations found with valid sentiment values. Cannot create map.")
+        return None
+    
+    # Group by state and calculate average sentiment
+    sentiment_by_state = df_with_locations.groupby('extracted_location')[sentiment_column].agg(
+        avg_sentiment=('mean'),
+        count=('count')
+    ).reset_index()
+    
+    return sentiment_by_state
+
+# Function to create India sentiment map
+def create_india_sentiment_map(sentiment_data, geojson_data, newspaper_name):
+    """
+    Create a choropleth map of India showing sentiment by state
+    """
+    # Ensure state names match between GeoJSON and our data
+    state_property = 'NAME_1'  # This is the property name in the GeoJSON
+    
+    # Determine color scale range based on data
+    min_sentiment = sentiment_data['avg_sentiment'].min()
+    max_sentiment = sentiment_data['avg_sentiment'].max()
+    
+    # Use symmetrical range if sentiment ranges from negative to positive
+    if min_sentiment < 0 and max_sentiment > 0:
+        abs_max = max(abs(min_sentiment), abs(max_sentiment))
+        color_range = [-abs_max, abs_max]
+    else:
+        # Add small buffer to range
+        color_range = [min_sentiment - 0.1, max_sentiment + 0.1]
+    
+    # Create the choropleth map
+    fig = px.choropleth_mapbox(
+        sentiment_data,
+        geojson=geojson_data,
+        locations='extracted_location',
+        featureidkey=f"properties.{state_property}",
+        color='avg_sentiment',
+        color_continuous_scale="RdBu",
+        range_color=color_range,
+        mapbox_style="carto-positron",
+        zoom=3.5,
+        center={"lat": 20.5937, "lon": 78.9629},
+        opacity=0.7,
+        hover_data=['count'],
+        labels={
+            'avg_sentiment': 'Average Sentiment',
+            'extracted_location': 'State',
+            'count': 'Article Count'
+        }
+    )
+    
+    # Customize the layout
+    fig.update_layout(
+        title=dict(
+            text=f'{newspaper_name} - Sentiment Analysis by Indian States',
+            font=dict(size=24, color='#2c3e50'),
+            x=0.5,
+            y=0.95
+        ),
+        height=800,
+        margin={"r":0,"t":50,"l":0,"b":0}
+    )
+    
+    # Add text annotation explaining the color scale
+    fig.add_annotation(
+        x=0.5, y=0.02,
+        xref="paper", yref="paper",
+        text="Color scale: Red (Negative) to Blue (Positive)",
+        showarrow=False,
+        font=dict(size=14)
+    )
+    
+    return fig
+
+# Function to plot sentiment trends by year (from original code)
+def plot_sentiment_trends_by_year(df, newspaper_name):
+    # Set the style to a clean, modern look
+    plt.style.use('seaborn-v0_8-whitegrid')
+
+    # Custom font settings
+    plt.rcParams['font.family'] = 'sans-serif'
+    plt.rcParams['font.sans-serif'] = ['Arial', 'Helvetica', 'DejaVu Sans']
+    plt.rcParams['font.size'] = 11
+    plt.rcParams['axes.titlesize'] = 16
+    plt.rcParams['axes.labelsize'] = 12
+
+    # Convert date to datetime and extract year
+    df['year'] = pd.to_datetime(df['date'], errors='coerce').dt.year
+
+    # Ensure only known sentiments are used
+    valid_sentiments = {"positive", "negative", "neutral"}
+    df['sentiment'] = df['sentiment_value'].apply(lambda x: x.lower() if isinstance(x, str) and x.lower() in valid_sentiments else "neutral")
+
+    # Count the number of articles per sentiment per year
+    sentiment_counts = df.groupby(['year', 'sentiment']).size().reset_index(name='count')
+
+    # Calculate total articles per year
+    year_totals = sentiment_counts.groupby('year')['count'].sum().reset_index(name='total')
+
+    # Merge the counts with totals to calculate percentages
+    sentiment_counts = sentiment_counts.merge(year_totals, on='year')
+    sentiment_counts['percentage'] = sentiment_counts['count'] / sentiment_counts['total'] * 100
+
+    # Pivot the data for easier plotting
+    sentiment_pivot = sentiment_counts.pivot(index='year', columns='sentiment', values='percentage').fillna(0)
+
+    # Ensure all sentiment columns exist
+    for sentiment in ['negative', 'neutral', 'positive']:
+        if sentiment not in sentiment_pivot.columns:
+            sentiment_pivot[sentiment] = 0
+
+    # Sort by year (ascending for timeline)
+    sentiment_pivot = sentiment_pivot.sort_index()
+
+    # Create the figure and axis
+    fig, ax = plt.subplots(figsize=(12, 7))
+
+    # Define custom colors
+    colors = {
+        'negative': '#5D3FD3',  # rich purple
+        'neutral': '#9D4EDD',   # lavender purple
+        'positive': '#00897B'   # teal green
+    }
+
+    # Plot lines for each sentiment
+    for sentiment in ['negative', 'neutral', 'positive']:
+        ax.plot(
+            sentiment_pivot.index,
+            sentiment_pivot[sentiment],
+            marker='o',
+            linewidth=2.5,
+            label=sentiment.capitalize(),
+            color=colors[sentiment],
+            markersize=8,
+            markeredgecolor='white',
+            markeredgewidth=1.5
+        )
+
+    # Add article counts as annotations
+    for year in sentiment_pivot.index:
+        total = year_totals.loc[year_totals['year'] == year, 'total'].values[0]
+        ax.annotate(
+            f"{total:,}",
+            xy=(year, sentiment_pivot.loc[year, 'negative'] - 5),
+            xytext=(0, -25),
+            textcoords='offset points',
+            ha='center',
+            fontsize=9,
+            color='gray'
+        )
+
+    # Add a text indicating what the numbers represent
+    ax.text(
+        sentiment_pivot.index[0],
+        -12,
+        "Article Count",
+        fontsize=9,
+        color='gray',
+        ha='center'
+    )
+
+    # Set x-axis to only show years (integers)
+    ax.xaxis.set_major_locator(MaxNLocator(integer=True))
+
+    # Set y-axis limits and labels
+    ax.set_ylim(0, max(100, sentiment_pivot.max().max() * 1.1))
+    ax.set_ylabel('Percentage (%)', fontweight='bold')
+    ax.set_xlabel('Year', fontweight='bold')
+
+    # Add title
+    ax.set_title(f'{newspaper_name} - Sentiment Trends by Year', fontweight='bold', pad=20)
+
+    # Customize legend
+    legend = ax.legend(
+        loc='upper right',
+        frameon=True,
+        framealpha=0.95,
+        edgecolor='lightgray',
+        title='Sentiment'
+    )
+    legend.get_title().set_fontweight('bold')
+
+    # Remove spines for cleaner look
+    ax.spines['top'].set_visible(False)
+    ax.spines['right'].set_visible(False)
+    ax.spines['left'].set_linewidth(0.5)
+    ax.spines['bottom'].set_linewidth(0.5)
+
+    # Add grid lines
+    ax.grid(axis='y', linestyle='--', alpha=0.3, color='gray')
+
+    # Add subtle background color
+    fig.patch.set_facecolor('#F8F9FA')
+    ax.set_facecolor('#F8F9FA')
+
+    # Add percentage labels at the end of each line
+    last_year = sentiment_pivot.index[-1]
+    for sentiment in ['negative', 'neutral', 'positive']:
+        if last_year in sentiment_pivot.index:  # Check if the last_year exists in the index
+            last_value = sentiment_pivot.loc[last_year, sentiment]
+            ax.annotate(
+                f"{last_value:.1f}%",
+                xy=(last_year, last_value),
+                xytext=(5, 0),
+                textcoords='offset points',
+                fontweight='bold',
+                color=colors[sentiment]
+            )
+
+    # Add a data source footer
+    plt.figtext(
+        0.01, 0.01,
+        f"Data source: Analysis of {df.shape[0]:,} articles",
+        fontsize=8,
+        color='gray'
+    )
+
+    # Add horizontal line at 50% for reference
+    ax.axhline(y=50, color='gray', linestyle='-', alpha=0.2)
+    ax.text(sentiment_pivot.index[0], 51, "50%", fontsize=8, color='gray')
+
+    # Adjust layout
+    plt.tight_layout(pad=2.0)
+
+    return fig
+
+# Function to plot article volume by year (from original code)
+def plot_article_volume_by_year(df, newspaper_name):
+    # Set the style to a clean, modern look
+    plt.style.use('seaborn-v0_8-whitegrid')
+
+    # Custom font settings
+    plt.rcParams['font.family'] = 'sans-serif'
+    plt.rcParams['font.sans-serif'] = ['Arial', 'Helvetica', 'DejaVu Sans']
+
+    # Convert date to datetime and extract year
+    df['year'] = pd.to_datetime(df['date'], errors='coerce').dt.year
+
+    # Count articles per year
+    article_counts = df.groupby('year').size().reset_index(name='count')
+
+    # Create the figure and axis
+    fig, ax = plt.subplots(figsize=(12, 5))
+
+    # Plot line for article count
+    ax.plot(
+        article_counts['year'],
+        article_counts['count'],
+        marker='o',
+        linewidth=2.5,
+        color='#3949AB',
+        markersize=8,
+        markeredgecolor='white',
+        markeredgewidth=1.5
+    )
+
+    # Fill area under the line
+    ax.fill_between(
+        article_counts['year'],
+        article_counts['count'],
+        alpha=0.2,
+        color='#3949AB'
+    )
+
+    # Set x-axis to only show years (integers)
+    ax.xaxis.set_major_locator(MaxNLocator(integer=True))
+
+    # Add count labels above each point
+    for year, count in zip(article_counts['year'], article_counts['count']):
+        ax.annotate(
+            f"{count:,}",
+            xy=(year, count),
+            xytext=(0, 10),
+            textcoords='offset points',
+            ha='center',
+            fontweight='bold',
+            fontsize=10
+        )
+
+    # Set axis labels
+    ax.set_ylabel('Number of Articles', fontweight='bold')
+    ax.set_xlabel('Year', fontweight='bold')
+
+    # Add title
+    ax.set_title(f'{newspaper_name} - Article Volume by Year', fontweight='bold', pad=20)
+
+    # Remove spines for cleaner look
+    ax.spines['top'].set_visible(False)
+    ax.spines['right'].set_visible(False)
+
+    # Add grid lines
+    ax.grid(axis='y', linestyle='--', alpha=0.3, color='gray')
+
+    # Add subtle background color
+    fig.patch.set_facecolor('#F8F9FA')
+    ax.set_facecolor('#F8F9FA')
+
+    # Adjust layout
+    plt.tight_layout()
+
+    return fig
+
+# Function to create a comparison bar chart of newspapers
+def create_newspaper_comparison(dataframes, newspaper_names):
+    # Prepare data for the comparison
+    comparison_data = []
+    
+    for i, df in enumerate(dataframes):
+        if df is not None:
+            # Ensure sentiment column exists and is properly formatted
+            if 'sentiment_value' in df.columns:
+                df['sentiment'] = df['sentiment_value'].apply(
+                    lambda x: x.lower() if isinstance(x, str) and x.lower() in ["positive", "negative", "neutral"] else "neutral"
+                )
+            
+            # Count articles by sentiment
+            sentiment_counts = df['sentiment'].value_counts().to_dict()
+            
+            # Add counts to comparison data
+            for sentiment in ['positive', 'negative', 'neutral']:
+                comparison_data.append({
+                    'Newspaper': newspaper_names[i],
+                    'Sentiment': sentiment.capitalize(),
+                    'Count': sentiment_counts.get(sentiment, 0)
+                })
+    
+    # Create DataFrame from comparison data
+    comparison_df = pd.DataFrame(comparison_data)
+    
+    # Create grouped bar chart
+    fig = px.bar(
+        comparison_df,
+        x='Newspaper',
+        y='Count',
+        color='Sentiment',
+        barmode='group',
+        title='Sentiment Distribution Across Newspapers',
+        color_discrete_map={
+            'Positive': '#00897B',
+            'Neutral': '#9D4EDD',
+            'Negative': '#5D3FD3'
+        }
+    )
+    
+    fig.update_layout(
+        height=500,
+        legend_title='Sentiment',
+        xaxis_title='',
+        yaxis_title='Number of Articles'
+    )
+    
+    return fig
+
+# Function to create a top locations bar chart
+def create_top_locations_chart(df, newspaper_name):
+    """Create a bar chart of the top mentioned locations"""
+    if 'extracted_location' not in df.columns or df['extracted_location'].isna().all():
+        # Return an empty figure
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No location data available",
+            showarrow=False,
+            font=dict(size=20)
+        )
+        fig.update_layout(height=400)
+        return fig
+    
+    # Count articles by location
+    location_counts = df['extracted_location'].value_counts().reset_index()
+    location_counts.columns = ['Location', 'Article Count']
+    
+    # Get top 15 locations
+    top_locations = location_counts.head(15)
+    
+    # Create bar chart
+    fig = px.bar(
+        top_locations,
+        y='Location',
+        x='Article Count',
+        title=f'Top 15 Locations Mentioned in {newspaper_name} Articles',
+        orientation='h',
+        color='Article Count',
+        color_continuous_scale='Viridis'
+    )
+    
+    fig.update_layout(
+        height=500,
+        yaxis={'categoryorder':'total ascending'}
+    )
+    
+    return fig
+
+def create_top_politicians_chart(df, newspaper_name):
+    """Create a bar chart of the top mentioned locations"""
+    
+    if 'Politician' not in df.columns or df['Politician'].isna().all():
+        print(df.head())
+        # Return an empty figure
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No politician data  available",
+            showarrow=False,
+            font=dict(size=20)
+        )
+        fig.update_layout(height=400)
+        return fig
+    
+    # Get top 15 locations
+    top_locations = df
+    
+    # Create bar chart
+    fig = px.bar(
+        top_locations,
+        y='Politician',
+        x='Mentions',
+        title=f'Top 10 Politicians Mentioned in {newspaper_name} Articles',
+        orientation='h',
+        color='Mentions',
+        color_continuous_scale='Viridis'
+    )
+    
+    fig.update_layout(
+        height=500,
+        yaxis={'categoryorder':'total ascending'}
+    )
+    
+    return fig
+
+# Function to load data
+@st.cache_data
+def load_data(newspaper_name):
+    try:
+        # Try to load CSV file for the newspaper
+        file_path = f"data/{newspaper_name.lower().replace(' ', '_')}_articles.csv"
+        df = pd.read_csv(file_path)
+        
+        # Check if required columns exist
+        required_columns = ['date', 'sentiment_value']
+        for col in required_columns:
+            if col not in df.columns:
+                st.error(f"Required column '{col}' not found in {file_path}")
+                return None
+        
+        # Add sentiment score column if not exists
+        if 'sentiment_score' not in df.columns:
+            # Create a numeric sentiment score based on sentiment_value
+            sentiment_map = {
+                'positive': 1.0,
+                'negative': -1.0,
+                'neutral': 0.0
+            }
+            df['sentiment_score'] = df['sentiment_value'].str.lower().map(sentiment_map).fillna(0)
+        
+        return df
+    
+    except Exception as e:
+        st.error(f"Error loading data for {newspaper_name}: {str(e)}")
+        return None
+
+# Entry page
+def show_entry_page():
+    st.markdown('<div class="entry-page">', unsafe_allow_html=True)
+    st.markdown('<div class="entry-container">', unsafe_allow_html=True)
+    
+    st.markdown('<h1 class="main-header">Sentiment Analysis of RSS Articles</h1>', unsafe_allow_html=True)
+    st.markdown("""
+    <p style="font-size: 1.2rem; margin-bottom: 2rem;">
+        Analyze the sentiments of news articles from various RSS feeds across different newspapers.
+        Discover trends, patterns, and insights through interactive visualizations.
+    </p>
+    """, unsafe_allow_html=True)
+    
+    st.markdown('<div class="button-container">', unsafe_allow_html=True)
+    if st.button("Explore Analysis", key="entry_explore", use_container_width=True):
+        st.session_state.show_entry = False
+    st.markdown('</div>', unsafe_allow_html=True)
+    
+    st.markdown('</div>', unsafe_allow_html=True)
+    st.markdown('</div>', unsafe_allow_html=True)
+
+# Home page with newspaper cards
+def show_home_page():
+    st.markdown('<h1 class="main-header">RSS Articles Sentiment Analysis Dashboard</h1>', unsafe_allow_html=True)
+    
+    # List of newspapers
+    newspapers = ["Print", "Scroll", "Sentinel", "NDTV"]
+    
+    # Load data for all newspapers
+    dataframes = []
+    for newspaper in newspapers:
+        df = load_data(newspaper)
+        dataframes.append(df)
+    
+    # Show comparison chart of all newspapers
+    st.markdown('<h2 class="sub-header">Newspaper Sentiment Comparison</h2>', unsafe_allow_html=True)
+    comparison_fig = create_newspaper_comparison(dataframes, newspapers)
+    st.plotly_chart(comparison_fig, use_container_width=True)
+    
+    # Create a 2x2 grid for newspaper cards
+    col1, col2 = st.columns(2)
+    col3, col4 = st.columns(2)
+    cols = [col1, col2, col3, col4]
+    
+    # Create a card for each newspaper
+    for i, newspaper in enumerate(newspapers):
+        df = dataframes[i]
+        with cols[i]:
+            st.markdown(f'<div class="newspaper-card">', unsafe_allow_html=True)
+            st.markdown(f'<div class="newspaper-title">{newspaper}</div>', unsafe_allow_html=True)
+            
+            # Only show counts if data is available
+            if df is not None:
+                # Count articles by sentiment
+                if 'sentiment_value' in df.columns:
+                    sentiment_counts = df['sentiment_value'].str.lower().value_counts()
+                    
+                    # Create three columns for sentiment counts
+                    pos_col, neu_col, neg_col = st.columns(3)
+                    with pos_col:
+                        st.metric("Positive", sentiment_counts.get('positive', 0))
+                    with neu_col:
+                        st.metric("Neutral", sentiment_counts.get('neutral', 0))
+                    with neg_col:
+                        st.metric("Negative", sentiment_counts.get('negative', 0))
+                else:
+                    st.write("Sentiment data not available")
+            else:
+                st.write("Data not available")
+            
+            # Add button to view detailed analysis
+            if st.button(f"View Analysis", key=f"view_{newspaper}"):
+                st.session_state.current_newspaper = newspaper
+                st.session_state.show_newspaper_analysis = True
+                st.rerun()
+            
+            st.markdown('</div>', unsafe_allow_html=True)
+
+# Function to process all newspapers with location extraction
+@st.cache_data
+def preprocess_newspapers_with_locations(newspapers):
+    # Load GeoJSON for India map
+    india_geojson = load_india_geojson()
+    if india_geojson is None:
+        st.error("Could not load India GeoJSON. Please check your internet connection.")
+        return {}
+    
+    processed_data = {}
+    
+    for newspaper in newspapers:
+        # Load the raw data
+        df = load_data(newspaper)
+        
+        if df is not None and 'desc' in df.columns:
+            # Extract locations if not already done
+            if 'extracted_location' not in df.columns:
+                df = extract_locations_from_descriptions(df, 'desc')
+            
+            # Analyze sentiment by state
+            sentiment_by_state = analyze_sentiment_by_state(df)
+            
+            processed_data[newspaper] = {
+                'df': df,
+                'sentiment_by_state': sentiment_by_state,
+                'india_geojson': india_geojson
+            }
+        else:
+            if df is not None:
+                processed_data[newspaper] = {
+                    'df': df,
+                    'error': "Description column 'desc' not found"
+                }
+            else:
+                processed_data[newspaper] = {
+                    'error': f"Could not load data for {newspaper}"
+                }
+    
+    return processed_data
+
+# Newspaper analysis page
+# Newspaper analysis page
+def show_newspaper_analysis():
+    # Add back button
+    if st.button("← Back to Home"):
+        st.session_state.show_newspaper_analysis = False
+        st.rerun()
+    
+    newspaper = st.session_state.current_newspaper
+    st.markdown(f'<h1 class="main-header">{newspaper} - Sentiment Analysis</h1>', unsafe_allow_html=True)
+    
+    # Load data for this newspaper
+    df = load_data(newspaper)
+    
+    if df is not None:
+        # Get or preprocess location data
+        if 'processed_data' not in st.session_state:
+            with st.spinner("Processing newspaper data..."):
+                st.session_state.processed_data = preprocess_newspapers_with_locations(["Print", "Scroll", "Sentinel", "NDTV"])
+        
+        processed_data = st.session_state.processed_data.get(newspaper, {})
+        
+        # Display article count and date range
+        article_count = len(df)
+        
+        # Convert date column to datetime to get min and max dates
+        df['date'] = pd.to_datetime(df['date'], errors='coerce')
+        min_date = df['date'].min().strftime('%d %b, %Y') if not pd.isna(df['date'].min()) else "Unknown"
+        max_date = df['date'].max().strftime('%d %b, %Y') if not pd.isna(df['date'].max()) else "Unknown"
+        
+        # Create metrics row
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            st.metric("Total Articles", f"{article_count:,}")
+        with col2:
+            st.metric("First Article", min_date)
+        with col3:
+            st.metric("Latest Article", max_date)
+        
+        # Show sentiment trends by year
+        st.markdown('<h2 class="sub-header">Sentiment Trends Over Time</h2>', unsafe_allow_html=True)
+        try:
+            sentiment_trend_fig = plot_sentiment_trends_by_year(df, newspaper)
+            st.pyplot(sentiment_trend_fig)
+        except Exception as e:
+            st.error(f"Error generating sentiment trends chart: {str(e)}")
+        
+        # Show article volume by year
+        st.markdown('<h2 class="sub-header">Article Volume by Year</h2>', unsafe_allow_html=True)
+        try:
+            volume_fig = plot_article_volume_by_year(df, newspaper)
+            st.pyplot(volume_fig)
+        except Exception as e:
+            st.error(f"Error generating article volume chart: {str(e)}")
+        
+        # Create two columns for location analysis
+        col1, col2 = st.columns(2)
+
+        with col1:
+            # Top mentioned locations
+            st.markdown('<h2 class="sub-header">Top Mentioned Locations</h2>', unsafe_allow_html=True)
+            
+            if 'extracted_location' in df.columns:
+                top_locations_fig = create_top_locations_chart(df, newspaper)
+                st.plotly_chart(top_locations_fig, use_container_width=True)
+            else:
+                if 'desc' in df.columns:
+                    st.info("Location data not yet extracted. Click the button below to extract locations.")
+                    if st.button("Extract Locations", key=f"extract_{newspaper}"):
+                        with st.spinner("Extracting locations..."):
+                            df = extract_locations_from_descriptions(df)
+                            # Update the processed data
+                            processed_data['df'] = df
+                            sentiment_by_state = analyze_sentiment_by_state(df)
+                            processed_data['sentiment_by_state'] = sentiment_by_state
+                            st.session_state.processed_data[newspaper] = processed_data
+                            st.experimental_rerun()
+                else:
+                    st.warning("Description column not found. Cannot extract locations.")
+            
+            # Top mentioned politicians - Now placed below the locations graph in the same column
+            st.markdown('<h2 class="sub-header">Top Mentioned Politicians</h2>', unsafe_allow_html=True)
+            
+            if 'desc' in df.columns:
+                # Check if rss_personalities is defined, if not you'll need to define it
+                if 'rss_personalities' not in locals() and 'rss_personalities' not in globals():
+                    # Define your list of politicians here or import it
+                    rss_personalities = ["Narendra Modi", "Amit Shah", "Rajnath Singh", "Mohan Bhagwat", "Yogi Adityanath", "Nitin Gadkari"]
+                
+                top_politicians = pussy.count_politicians_in_descriptions(df, rss_personalities).head(10)
+                top_politicians_fig = create_top_politicians_chart(top_politicians, newspaper)
+                st.plotly_chart(top_politicians_fig, use_container_width=True)
+            else:
+                st.warning("Description column not found. Cannot analyze politicians.")
+
+        with col2:
+            # Sentiment by state map
+            st.markdown('<h2 class="sub-header">Sentiment by State</h2>', unsafe_allow_html=True)
+            
+            sentiment_by_state = processed_data.get('sentiment_by_state')
+            india_geojson = processed_data.get('india_geojson')
+            
+            if sentiment_by_state is not None and india_geojson is not None and not sentiment_by_state.empty:
+                try:
+                    map_fig = create_india_sentiment_map(sentiment_by_state, india_geojson, newspaper)
+                    st.plotly_chart(map_fig, use_container_width=True)
+                except Exception as e:
+                    st.error(f"Error creating sentiment map: {str(e)}")
+            else:
+                if 'error' in processed_data:
+                    st.warning(processed_data['error'])
+                else:
+                    st.info("Sentiment data not available. Extract locations first.")
+
+        # Add section for detailed article analysis
+        st.markdown('<h2 class="sub-header">Article Analysis</h2>', unsafe_allow_html=True)
+                
+        # Add filters for article display
+        col1, col2, col3 = st.columns(3)
+        
+        with col1:
+            # Sentiment filter
+            sentiment_options = ["All"] + sorted(df['sentiment_value'].unique().tolist())
+            selected_sentiment = st.selectbox("Filter by Sentiment", sentiment_options)
+        
+        with col2:
+            # Year filter
+            year_options = ["All"] + sorted(df['date'].dt.year.dropna().unique().astype(int).tolist())
+            selected_year = st.selectbox("Filter by Year", year_options)
+        
+        with col3:
+            # Location filter (if available)
+            location_options = ["All"]
+            if 'extracted_location' in df.columns:
+                location_options += sorted(df['extracted_location'].dropna().unique().tolist())
+            selected_location = st.selectbox("Filter by Location", location_options)
+        
+        # Apply filters
+        filtered_df = df.copy()
+        
+        if selected_sentiment != "All":
+            filtered_df = filtered_df[filtered_df['sentiment_value'] == selected_sentiment]
+        
+        if selected_year != "All":
+            filtered_df = filtered_df[filtered_df['date'].dt.year == selected_year]
+        
+        if selected_location != "All" and 'extracted_location' in filtered_df.columns:
+            filtered_df = filtered_df[filtered_df['extracted_location'] == selected_location]
+        
+        # Show article count after filtering
+        st.write(f"Displaying {len(filtered_df)} articles based on your filters.")
+        
+        # Display articles in an expandable format
+        if not filtered_df.empty:
+            for index, row in filtered_df.head(50).iterrows():
+                title = row.get('title', 'Untitled')
+                date = row['date'].strftime('%d %b, %Y') if pd.notna(row['date']) else 'Unknown date'
+                sentiment = row.get('sentiment_value', 'Unknown sentiment')
+                description = row.get('desc', 'No description available')
+                link = row.get('link', 'No link available')
+                
+                # Format sentiment with color
+                sentiment_color = {
+                    'positive': 'green',
+                    'neutral': 'gray',
+                    'negative': 'red'
+                }.get(sentiment.lower(), 'gray')
+                
+                # Create expandable card for each article
+                with st.expander(f"{title} - {date}"):
+                    st.markdown(f"**Sentiment:** <span style='color:{sentiment_color}'>{sentiment.capitalize()}</span>", unsafe_allow_html=True)
+                    
+                    if 'extracted_location' in row and pd.notna(row['extracted_location']):
+                        st.markdown(f"**Location:** {row['extracted_location']}")
+                    
+                    st.markdown("**Description:**")
+                    st.markdown(f"{description}")
+                    st.markdown(f"**Link:**  {link}")
+            
+            if len(filtered_df) > 50:
+                st.info(f"Showing 50 out of {len(filtered_df)} articles. Apply more filters to narrow down results.")
+        else:
+            st.info("No articles match your selected filters.")
+    else:
+        st.error(f"Could not load data for {newspaper}. Please check if the data file exists.")
+
+# Main function
+def main():
+    # Load CSS
+    load_css()
+    
+    # Initialize session state variables if not exists
+    if 'show_entry' not in st.session_state:
+        st.session_state.show_entry = True
+    
+    if 'show_newspaper_analysis' not in st.session_state:
+        st.session_state.show_newspaper_analysis = False
+    
+    if 'current_newspaper' not in st.session_state:
+        st.session_state.current_newspaper = None
+    
+    # Display appropriate page based on session state
+    if st.session_state.show_entry:
+        show_entry_page()
+    elif st.session_state.show_newspaper_analysis:
+        show_newspaper_analysis()
+    else:
+        show_home_page()
+    
+    # Footer
+    st.markdown('<div class="footer">', unsafe_allow_html=True)
+    st.markdown('RSS Sentiment Analysis Dashboard - Developed with Streamlit', unsafe_allow_html=True)
+    st.markdown('</div>', unsafe_allow_html=True)
+
+if __name__ == "__main__":
+    main()

src/requirements.txt

@@ -0,0 +1,6 @@
+selenium
+beautifulsoup4
+requests
+webdriver-manager
+pandas
+tqdm

src/run_scraper.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""
+Main entry point for running the news scrapers.
+This script acts as a unified interface to run any of the available scrapers.
+"""
+
+import os
+import sys
+import argparse
+import importlib
+from datetime import datetime
+
+# Make sure the output directory exists
+os.makedirs('output', exist_ok=True)
+
+def parse_arguments():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description='Scrape news articles from Indian news websites')
+    
+    # Required arguments
+    parser.add_argument('--source', '-s', type=str, required=True,
+                        choices=['toi', 'ndtv', 'wion', 'scroll'],
+                        help='News source to scrape from')
+    parser.add_argument('--topic', '-t', type=str, required=True,
+                        help='Topic to search for (e.g., "Climate Change", "Politics")')
+    
+    # Optional arguments
+    parser.add_argument('--workers', '-w', type=int, default=4,
+                        help='Number of worker threads (default: 4)')
+    parser.add_argument('--interval', '-i', type=int, default=300,
+                        help='Auto-save interval in seconds (default: 300)')
+    
+    return parser.parse_args()
+
+def get_scraper_class(source):
+    """Dynamically import the appropriate scraper class based on the source"""
+    source_map = {
+        'toi': ('scrapers.toi_scraper', 'TOIArticleScraper'),
+        'ndtv': ('scrapers.ndtv_scraper', 'NDTVArticleScraper'),
+        'wion': ('scrapers.wion_scraper', 'WIONArticleScraper'),
+        'scroll': ('scrapers.scroll_scraper', 'ScrollArticleScraper')
+    }
+    
+    if source not in source_map:
+        raise ValueError(f"Unsupported news source: {source}")
+    
+    module_name, class_name = source_map[source]
+    module = importlib.import_module(module_name)
+    return getattr(module, class_name)
+
+def main():
+    """Main function to run the scraper based on command line arguments"""
+    args = parse_arguments()
+    
+    try:
+        print(f"\n--- Indian News Scraper ---")
+        print(f"Source: {args.source}")
+        print(f"Topic: {args.topic}")
+        print(f"Workers: {args.workers}")
+        print(f"Auto-save interval: {args.interval} seconds")
+        print("---------------------------\n")
+        
+        # Get the appropriate scraper class
+        ScrapeClass = get_scraper_class(args.source)
+        
+        # Initialize the scraper
+        scraper = ScrapeClass(max_workers=args.workers)
+        scraper.save_interval = args.interval
+        
+        # Configure output directory
+        os.chdir('output')
+        
+        print(f"Starting to scrape {args.topic}-related articles from {args.source.upper()}...")
+        print("Press Ctrl+C at any time to save progress and exit.")
+        
+        # Run the scraper (accounting for different parameter names in different scrapers)
+        if args.source == 'toi':
+            topic_url = f"{scraper.base_url}/topic/{args.topic}/news"
+            final_csv = scraper.scrape_topic(topic_url, args.topic)
+        elif args.source == 'ndtv':
+            final_csv = scraper.scrape_topic(args.topic)
+        elif args.source == 'wion' or args.source == 'scroll':
+            final_csv = scraper.scrape_topic(args.topic.lower(), args.topic)
+        
+        # Print results
+        if final_csv:
+            article_count = len(scraper.articles) if hasattr(scraper, 'articles') else len(scraper.scraped_articles)
+            print(f"\nArticles have been saved to: {final_csv}")
+            print(f"Total articles scraped: {article_count}")
+        else:
+            print("\nError saving to final CSV file")
+        
+    except KeyboardInterrupt:
+        print("\nProcess interrupted by user. Saving progress...")
+        articles = getattr(scraper, 'articles', None) or getattr(scraper, 'scraped_articles', [])
+        if articles:
+            scraper.save_progress(args.topic, force=True)
+        print("Saved progress and exiting.")
+    except Exception as e:
+        print(f"\nAn error occurred: {str(e)}")
+        articles = getattr(scraper, 'articles', None) or getattr(scraper, 'scraped_articles', [])
+        if 'scraper' in locals() and articles:
+            scraper.save_progress(args.topic, force=True)
+        print("Saved progress despite error.")
+        return 1
+        
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

src/scrapers/__init__.py

@@ -0,0 +1,6 @@
+"""
+News scraper package for Indian news websites.
+Contains scrapers for TOI, NDTV, WION, and Scroll.in.
+"""
+
+__version__ = '1.0.0'

src/scrapers/ndtv_scraper.py

@@ -0,0 +1,338 @@
+import os
+import time
+import logging
+import csv
+import signal
+import sys
+import argparse
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import (
+    TimeoutException, NoSuchElementException, ElementClickInterceptedException
+)
+from bs4 import BeautifulSoup
+from utils.webdriver_utils import create_chrome_driver, wait_for_page_load, scroll_to_element
+
+class NDTVArticleScraper:
+    def __init__(self, max_workers=4):
+        self.max_workers = max_workers
+        self.base_url = "https://www.ndtv.com/"
+        self.fetched_articles = set()
+        self.scraped_articles = []
+        self.last_save_time = time.time()
+        self.save_interval = 300  # Save every 5 minutes
+        self.is_interrupted = False
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger(__name__)
+
+        # Set up signal handlers
+        signal.signal(signal.SIGINT, self.signal_handler)
+        signal.signal(signal.SIGTERM, self.signal_handler)
+
+    def signal_handler(self, signum, frame):
+        """Handle interrupt signals"""
+        print("\nReceived interrupt signal. Saving progress and shutting down...")
+        self.is_interrupted = True
+        if self.scraped_articles:
+            self.save_progress("interrupted", force=True)
+        sys.exit(0)
+
+    # def create_driver(self):
+    #     """Create and return a new Chrome driver instance"""
+    #     chrome_options = webdriver.ChromeOptions()
+    #     chrome_options.add_argument('--headless')
+    #     chrome_options.add_argument('--no-sandbox')
+    #     chrome_options.add_argument('--disable-dev-shm-usage')
+    #     chrome_options.add_argument('--disable-extensions')
+    #     chrome_options.page_load_strategy = 'eager'
+    #     return webdriver.Chrome(options=chrome_options)
+    
+    def create_driver(self):
+        """Create and return a new Chrome driver instance"""
+        return create_chrome_driver(headless=True, load_images=False)
+
+    def scroll_to_element(self, driver, element):
+        try:
+            driver.execute_script("arguments[0].scrollIntoView(true);", element)
+            driver.execute_script("window.scrollBy(0, -100);")
+        except Exception as e:
+            self.logger.error(f"Error scrolling to element: {str(e)}")
+
+    def extract_visible_articles(self, driver):
+        """Extract articles currently visible on the page"""
+        soup = BeautifulSoup(driver.page_source, 'html.parser')
+        article_containers = soup.find_all('div', class_='SrchLstPg-a')
+
+        new_articles = []
+        for container in article_containers:
+            link = container.find('a', href=True)
+            date = self.extract_date(container)
+
+            if link:
+                full_link = link['href'] if link['href'].startswith('http') else self.base_url + link['href']
+                if full_link not in self.fetched_articles:
+                    self.fetched_articles.add(full_link)
+                    new_articles.append({'link': full_link, 'date': date})
+
+        self.logger.info(f"Found {len(new_articles)} new articles")
+        return new_articles
+
+    def save_progress(self, topic, force=False):
+        """Save current progress to CSV"""
+        current_time = time.time()
+        if force or (current_time - self.last_save_time >= self.save_interval and self.scraped_articles):
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"ndtv_{topic}articles_{timestamp}_partial.csv"
+            try:
+                with open(filename, 'w', newline='', encoding='utf-8') as file:
+                    writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                    writer.writeheader()
+                    writer.writerows(self.scraped_articles)
+                self.last_save_time = current_time
+                print(f"\nProgress saved to {filename}: {len(self.scraped_articles)} articles")
+                self.logger.info(f"Progress saved to {filename}: {len(self.scraped_articles)} articles")
+            except Exception as e:
+                self.logger.error(f"Error saving progress: {str(e)}")
+
+    def get_article_links(self, topic_url, topic):
+        driver = self.create_driver()
+        all_articles = []
+
+        try:
+            driver.get(topic_url)
+            time.sleep(2)
+
+            # First scrape visible articles
+            initial_articles = self.extract_visible_articles(driver)
+            all_articles.extend(initial_articles)
+            self.logger.info(f"Initially found {len(initial_articles)} visible articles")
+
+            # Process these articles first
+            if initial_articles:
+                self.process_articles_batch(initial_articles)
+                self.save_progress(topic)  # Save after initial batch
+
+            # Then continue with "Load More" and scrape new articles
+            attempt = 0
+            while not self.is_interrupted:
+                try:
+                    parent_div = WebDriverWait(driver, 5).until(
+                        EC.presence_of_element_located((By.CLASS_NAME, "btn_bm-cntr"))
+                    )
+                    load_more_button = parent_div.find_element(By.TAG_NAME, "a")
+                    self.scroll_to_element(driver, load_more_button)
+                    driver.execute_script("arguments[0].click();", load_more_button)
+                    attempt += 1
+                    print(f"Clicking load more {attempt} time")
+                    time.sleep(2)
+
+                    # Extract newly loaded articles
+                    new_articles = self.extract_visible_articles(driver)
+                    new_count = len(new_articles)
+                    print(f"Found {new_count} new articles after clicking load more")
+
+                    new_articles = [article for article in new_articles if article['link'] not in {a['link'] for a in all_articles}]
+                    all_articles.extend(new_articles)
+
+                    # Process new batch of articles
+                    if new_articles:
+                        self.process_articles_batch(new_articles)
+                        self.save_progress(topic)  # Save after each batch
+
+                except (NoSuchElementException, TimeoutException, ElementClickInterceptedException):
+                    self.logger.info("No more 'Load More' button found or it is no longer clickable.")
+                    break
+                except Exception as e:
+                    self.logger.error(f"Error during load more operation: {str(e)}")
+                    self.save_progress(topic, force=True)
+                    break
+
+            return all_articles
+
+        except Exception as e:
+            self.logger.error(f"Error getting article links: {str(e)}")
+            self.save_progress(topic, force=True)
+            return all_articles
+        finally:
+            driver.quit()
+
+    def process_articles_batch(self, articles):
+        """Process a batch of articles immediately"""
+        self.logger.info(f"Processing batch of {len(articles)} articles")
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.scrape_article_parallel, article) for article in articles]
+            for future in as_completed(futures):
+                try:
+                    article = future.result()
+                    if article:
+                        self.scraped_articles.append(article)
+                        self.logger.info(f"Successfully scraped: {article['title'][:50]}...")
+                except Exception as e:
+                    self.logger.error(f"Error processing article: {str(e)}")
+
+    def scrape_article_parallel(self, article_data):
+        """Scrape a single article with its own driver instance"""
+        driver = self.create_driver()
+        try:
+            url = article_data['link']
+            driver.get(url)
+
+            try:
+                title_element = WebDriverWait(driver, 5).until(
+                    EC.presence_of_element_located((By.TAG_NAME, "h1"))
+                )
+            except TimeoutException:
+                pass
+
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+
+            # Extract title
+            title = None
+            title_element = soup.find('h1')
+            if title_element:
+                title = title_element.get_text().strip()
+            desc = None
+            desc_element = soup.find('div', class_='Art-exp_wr')
+            if desc_element:
+
+                paragraphs = desc_element.find_all('p')
+                if paragraphs:
+                     desc = ' '.join([p.get_text().strip() for p in paragraphs])
+                else:
+                     desc = desc_element.get_text().strip()
+
+            # Clean up the description
+            if desc:
+                desc = desc.strip("<br>").strip("<!-- -->").strip('<div>').strip("\n")
+                # desc = desc_element.get_text().strip("<br>").strip("<!-- -->").strip('<div>').strip("\n")
+
+            if not title:
+                meta_title = soup.find('meta', property='og:title')
+                if meta_title:
+                    title = meta_title.get('content', '').strip()
+
+            return {
+                'title': title or 'Title not found',
+                'desc': desc or 'Description not found',
+                'date': article_data['date'] or 'Date not found',
+                'link': url
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error scraping article {url}: {str(e)}")
+            return None
+        finally:
+            driver.quit()
+
+    def extract_date(self, container):
+        try:
+            date_div = container.find('span', class_='pst-by_lnk')
+            if date_div:
+                contents = list(date_div.descendants)[0].split(" ")[1:]
+                date = ''
+                for content in contents:
+                    date += content + ' '
+                return date.rstrip(' ')
+            return None
+        except Exception as e:
+            self.logger.error(f"Error extracting date: {str(e)}")
+            return None
+
+    def scrape_topic(self, topic):
+        try:
+            topic_url = f"{self.base_url}/search?searchtext={topic}"
+            self.logger.info(f"Scraping topic: {topic_url}")
+
+            articles = self.get_article_links(topic_url, topic)
+            self.logger.info(f"Found total of {len(articles)} articles")
+
+            # Final save with complete data
+            if self.scraped_articles:
+                return self.save_to_csv(self.scraped_articles, topic, final=True)
+            return None
+
+        except Exception as e:
+            self.logger.error(f"Error in scrape_topic: {str(e)}")
+            if self.scraped_articles:
+                return self.save_to_csv(self.scraped_articles, topic, final=True)
+            return None
+
+    def save_to_csv(self, articles, topic, final=False):
+        if not articles:
+            self.logger.error("No articles to save")
+            return None
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"ndtv_{topic}articles_{timestamp}_{'final' if final else 'partial'}.csv"
+
+        try:
+            with open(filename, 'w', newline='', encoding='utf-8') as file:
+                writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                writer.writeheader()
+                writer.writerows(articles)
+                self.logger.info(f"Successfully saved {len(articles)} articles to: {filename}")
+            return filename
+        except Exception as e:
+            self.logger.error(f"Error saving to CSV: {str(e)}")
+            return None
+
+def parse_arguments():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description='Scrape articles from NDTV by topic')
+    
+    parser.add_argument('topic', type=str, 
+                        help='Topic to scrape (e.g., "RSS", "Covid", "India")')
+    
+    parser.add_argument('-w', '--workers', type=int, default=4,
+                        help='Number of worker threads (default: 4)')
+    
+    parser.add_argument('-i', '--interval', type=int, default=300,
+                        help='Auto-save interval in seconds (default: 300)')
+    
+    return parser.parse_args()
+
+def main():
+    try:
+        # Parse command line arguments
+        args = parse_arguments()
+        
+        # Initialize the scraper with command line arguments
+        scraper = NDTVArticleScraper(max_workers=args.workers)
+        scraper.save_interval = args.interval
+        
+        # Get topic from command line argument
+        topic = args.topic
+
+        print(f"\nScraping {topic}-related articles from NDTV...")
+        print("Press Ctrl+C at any time to save progress and exit.")
+
+        final_csv = scraper.scrape_topic(topic)
+
+        if final_csv:
+            print(f"\nArticles have been saved to: {final_csv}")
+            print(f"Total articles scraped: {len(scraper.scraped_articles)}")
+        else:
+            print("\nError saving to final CSV file")
+
+    except KeyboardInterrupt:
+        print("\nProcess interrupted by user. Saving progress...")
+        if scraper.scraped_articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress and exiting.")
+    except Exception as e:
+        print(f"\nAn error occurred: {str(e)}")
+        if 'scraper' in locals() and scraper.scraped_articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress despite error.")
+
+if __name__ == "__main__":
+    main()

src/scrapers/scroll_scraper.py

@@ -0,0 +1,314 @@
+import os
+import time
+import logging
+import csv
+import signal
+import sys
+import argparse
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import (
+    TimeoutException, NoSuchElementException, ElementClickInterceptedException
+)
+from bs4 import BeautifulSoup
+from utils.webdriver_utils import create_chrome_driver, scroll_to_element
+
+class ScrollArticleScraper:
+    def __init__(self, max_workers=4, articles_per_page=10):
+        self.max_workers = max_workers
+        self.articles_per_page = articles_per_page
+        self.base_url = "https://scroll.in/search"
+        self.fetched_articles = set()
+        self.articles = []
+        self.is_interrupted = False
+        self.last_save_time = time.time()
+        self.save_interval = 300  # Save every 5 minutes
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger(__name__)
+
+        # Set up signal handlers
+        signal.signal(signal.SIGINT, self.signal_handler)
+        signal.signal(signal.SIGTERM, self.signal_handler)
+
+    def signal_handler(self, signum, frame):
+        """Handle interrupt signals"""
+        print("\nReceived interrupt signal. Saving progress and shutting down...")
+        self.is_interrupted = True
+        if self.articles:
+            self.save_progress("interrupted", force=True)
+        sys.exit(0)
+
+    # def create_driver(self):
+    #     """Create and return a new Chrome driver instance"""
+    #     chrome_options = webdriver.ChromeOptions()
+    #     chrome_options.add_argument('--headless')
+    #     chrome_options.add_argument('--no-sandbox')
+    #     chrome_options.add_argument('--disable-dev-shm-usage')
+    #     chrome_options.add_argument('--disable-extensions')
+    #     chrome_options.page_load_strategy = 'eager'
+    #     return webdriver.Chrome(options=chrome_options)
+    def create_driver(self):
+        """Create and return a new Chrome driver instance"""
+        return create_chrome_driver(headless=True, load_images=False)
+    
+    def get_total_pages(self, driver, search_term):
+        """Get the total number of pages for the search term"""
+        try:
+            driver.get(f"{self.base_url}?q={search_term}&page=1")
+            time.sleep(2)
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+
+            # Scroll.in might have a different pagination structure
+            pagination = soup.find('div', class_='page-numbers')
+            if pagination:
+                pages = pagination.find_all('a', class_='page-number')
+                if pages:
+                    # Get the last page number
+                    last_page = max([int(page.text.strip()) for page in pages])
+                    return last_page
+
+            # Fallback to 144 pages if pagination not found
+            return 144
+        except Exception as e:
+            self.logger.error(f"Error getting total pages: {str(e)}")
+            return 144  # Default to 144 pages
+    def extract_visible_articles(self, driver):
+        soup = BeautifulSoup(driver.page_source, 'html.parser')
+        article_containers = soup.find_all('li', itemscope=True, itemtype="https://schema.org/NewsArticle")
+
+        new_articles = []
+        for container in article_containers:
+            link_element = container.find('a', href=True)
+            date_element = container.find('time')  # Change back to 'time'
+
+            if link_element and link_element['href']:
+                full_link = link_element['href'] if link_element['href'].startswith('http') else 'https://scroll.in' + link_element['href']
+
+                # Extract date
+                date = None
+                if date_element:
+                    date_text = date_element.get_text(strip=True)
+                    date_text = date_text.replace('Â', '').replace('·', '').strip()
+                    try:
+                        # Try parsing the date
+                        parsed_date = datetime.strptime(date_text, '%b %d, %Y')
+                        date = parsed_date.strftime('%Y-%m-%d')
+                    except:
+                        try:
+                            # Alternative date format
+                            parsed_date = datetime.strptime(date_text, '%B %d, %Y')
+                            date = parsed_date.strftime('%Y-%m-%d')
+                        except:
+                            date = date_text
+
+                if full_link not in self.fetched_articles:
+                    self.fetched_articles.add(full_link)
+                    new_articles.append({'link': full_link, 'date': date})
+
+        return new_articles
+
+    def scrape_topic(self, search_term, topic):
+        """Scrape articles from all pages for a given search term"""
+        driver = self.create_driver()
+        try:
+            total_pages = self.get_total_pages(driver, search_term)
+            total_expected_articles = total_pages * self.articles_per_page
+            self.logger.info(f"Found {total_pages} pages to scrape (approximately {total_expected_articles} articles)")
+
+            for page in range(1, total_pages + 1):
+                if self.is_interrupted:
+                    break
+
+                page_url = f"{self.base_url}?q={search_term}&page={page}"
+                articles_scraped = len(self.articles)
+                progress_percentage = (articles_scraped / total_expected_articles) * 100
+
+                self.logger.info(f"Scraping page {page}/{total_pages} - Progress: {articles_scraped}/{total_expected_articles} articles ({progress_percentage:.1f}%)")
+
+                try:
+                    driver.get(page_url)
+                    time.sleep(2)  # Allow page to load
+
+                    new_articles = self.extract_visible_articles(driver)
+                    if new_articles:
+                        self.process_articles_batch(new_articles)
+                        self.logger.info(f"Scraped {len(new_articles)}/{self.articles_per_page} articles from page {page}")
+                        self.save_progress(topic)
+                    else:
+                        self.logger.warning(f"No articles found on page {page}")
+
+                except Exception as e:
+                    self.logger.error(f"Error scraping page {page}: {str(e)}")
+                    continue
+
+            if self.articles:
+                return self.save_to_csv(self.articles, topic, final=True)
+            return None
+
+        except Exception as e:
+            self.logger.error(f"Error in scrape_topic: {str(e)}")
+            if self.articles:
+                return self.save_to_csv(self.articles, topic, final=True)
+            return None
+        finally:
+            driver.quit()
+
+    def scrape_article_parallel(self, article_data):
+
+        driver = self.create_driver()
+        try:
+            url = article_data['link']
+            driver.get(url)
+            time.sleep(2)
+
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+
+            # Extract title
+            title = None
+            title_element = soup.find('h1')  # Simple h1 search
+            if title_element:
+                title = title_element.get_text().strip()
+
+            # Extract description
+            desc = None
+            article_body = soup.find('section', class_='article-content')
+            if article_body:
+                paragraphs = article_body.find_all('p')
+                desc = ' '.join([p.get_text().strip() for p in paragraphs])
+
+            return {
+                'title': title or 'Title not found',
+                'desc': desc or 'Description not found',
+                'date': article_data['date'] or 'Date not found',
+                'link': url
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error scraping article {url}: {str(e)}")
+            return None
+        finally:
+            driver.quit()
+
+    def process_articles_batch(self, article_batch):
+        """Process a batch of articles in parallel"""
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.scrape_article_parallel, article_data)
+                      for article_data in article_batch]
+
+            successful_articles = 0
+            for future in as_completed(futures):
+                try:
+                    article = future.result()
+                    if article:
+                        self.articles.append(article)
+                        successful_articles += 1
+                        self.logger.info(f"Successfully scraped: {article['title'][:50]}...")
+                except Exception as e:
+                    self.logger.error(f"Error processing article: {str(e)}")
+
+            if successful_articles < len(article_batch):
+                self.logger.warning(f"Only processed {successful_articles}/{len(article_batch)} articles in this batch")
+
+    def save_progress(self, topic, force=False):
+        """Save current progress to CSV"""
+        current_time = time.time()
+        if force or (current_time - self.last_save_time >= self.save_interval and self.articles):
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"scroll_{topic}articles_{timestamp}_partial.csv"
+            try:
+                with open(filename, 'w', newline='', encoding='utf-8') as file:
+                    writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                    writer.writeheader()
+                    writer.writerows(self.articles)
+                self.last_save_time = current_time
+                print(f"\nProgress saved to {filename}: {len(self.articles)} articles")
+                self.logger.info(f"Progress saved to {filename}: {len(self.articles)} articles")
+            except Exception as e:
+                self.logger.error(f"Error saving progress: {str(e)}")
+
+    def save_to_csv(self, articles, topic, final=False):
+        """Save articles to CSV file"""
+        if not articles:
+            self.logger.error("No articles to save")
+            return None
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"scroll_{topic}articles_{timestamp}_{'final' if final else 'partial'}.csv"
+
+        try:
+            with open(filename, 'w', newline='', encoding='utf-8') as file:
+                writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                writer.writeheader()
+                writer.writerows(articles)
+                self.logger.info(f"Successfully saved {len(articles)} articles to: {filename}")
+            return filename
+        except Exception as e:
+            self.logger.error(f"Error saving to CSV: {str(e)}")
+            return None
+
+# def parse_arguments():
+#     """Parse command line arguments"""
+#     parser = argparse.ArgumentParser(description='Scrape articles from Scroll.in by topic')
+    
+#     parser.add_argument('topic', type=str, 
+#                         help='Topic to scrape (e.g., "RSS", "Covid", "India")')
+    
+#     parser.add_argument('-w', '--workers', type=int, default=4,
+#                         help='Number of worker threads (default: 4)')
+    
+#     parser.add_argument('-i', '--interval', type=int, default=300,
+#                         help='Auto-save interval in seconds (default: 300)')
+    
+#     parser.add_argument('-a', '--articles-per-page', type=int, default=10,
+#                         help='Expected number of articles per page (default: 10)')
+    
+#     return parser.parse_args()
+
+def main():
+    try:
+        # Parse command line arguments
+        args = parse_arguments()
+        
+        # Initialize the scraper with command line arguments
+        scraper = ScrollArticleScraper(
+            max_workers=args.workers,
+            articles_per_page=args.articles_per_page
+        )
+        scraper.save_interval = args.interval
+        
+        # Get topic from command line argument
+        topic = args.topic
+
+        print(f"\nScraping {topic}-related articles from Scroll.in...")
+        print("Press Ctrl+C at any time to save progress and exit.")
+
+        final_csv = scraper.scrape_topic(topic.lower(), topic)
+
+        if final_csv:
+            print(f"\nArticles have been saved to: {final_csv}")
+            print(f"Total articles scraped: {len(scraper.articles)}")
+        else:
+            print("\nError saving to final CSV file")
+
+    except KeyboardInterrupt:
+        print("\nProcess interrupted by user. Saving progress...")
+        if scraper.articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress and exiting.")
+    except Exception as e:
+        print(f"\nAn error occurred: {str(e)}")
+        if 'scraper' in locals() and scraper.articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress despite error.")
+
+if __name__ == "__main__":
+    main()

src/scrapers/toi_scraper.py

@@ -0,0 +1,309 @@
+import os
+import time
+import logging
+import csv
+import signal
+import sys
+import argparse
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import (
+    TimeoutException, NoSuchElementException, ElementClickInterceptedException
+)
+from bs4 import BeautifulSoup
+from utils.webdriver_utils import create_chrome_driver, scroll_to_element
+
+class TOIArticleScraper:
+    def __init__(self, max_workers=4):
+        self.max_workers = max_workers
+        self.base_url = "https://timesofindia.indiatimes.com"
+        self.fetched_articles = set()
+        self.articles = []
+        self.is_interrupted = False
+        self.last_save_time = time.time()
+        self.save_interval = 300  # Save every 5 minutes
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger(__name__)
+
+        # Set up signal handlers
+        signal.signal(signal.SIGINT, self.signal_handler)
+        signal.signal(signal.SIGTERM, self.signal_handler)
+
+    def create_driver(self):
+        """Create and return a new Chrome driver instance"""
+        return create_chrome_driver(headless=True, load_images=False)
+
+    def signal_handler(self, signum, frame):
+        """Handle interrupt signals"""
+        print("\nReceived interrupt signal. Saving progress and shutting down...")
+        self.is_interrupted = True
+        if self.articles:
+            self.save_progress("interrupted", force=True)
+        sys.exit(0)
+
+    # def create_driver(self):
+    #     """Create and return a new Chrome driver instance"""
+    #     chrome_options = webdriver.ChromeOptions()
+    #     chrome_options.add_argument('--headless')
+    #     chrome_options.add_argument('--no-sandbox')
+    #     chrome_options.add_argument('--disable-dev-shm-usage')
+    #     chrome_options.add_argument('--disable-extensions')
+    #     chrome_options.page_load_strategy = 'eager'
+    #     return webdriver.Chrome(options=chrome_options)
+    
+    def create_driver(self):
+        """Create and return a new Chrome driver instance"""
+        return create_chrome_driver(headless=True, load_images=False)
+
+    def scroll_to_element(self, driver, element):
+        """Scroll to make the element visible"""
+        try:
+            driver.execute_script("arguments[0].scrollIntoView(true);", element)
+            driver.execute_script("window.scrollBy(0, -100);")
+        except Exception as e:
+            self.logger.error(f"Error scrolling to element: {str(e)}")
+
+    def extract_visible_articles(self, driver):
+        """Extract currently visible articles from the page"""
+        soup = BeautifulSoup(driver.page_source, 'html.parser')
+        article_containers = soup.find_all('div', class_='uwU81')
+
+        new_articles = []
+        for container in article_containers:
+            link = container.find('a', href=True)
+            date = self.extract_date(container)
+
+            if link and 'articleshow' in link['href']:
+                full_link = link['href'] if link['href'].startswith('http') else self.base_url + link['href']
+                if full_link not in self.fetched_articles:
+                    self.fetched_articles.add(full_link)
+                    new_articles.append({'link': full_link, 'date': date})
+
+        return new_articles
+
+    def save_progress(self, topic, force=False):
+        """Save current progress to CSV"""
+        current_time = time.time()
+        if force or (current_time - self.last_save_time >= self.save_interval and self.articles):
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"toi_{topic}articles_{timestamp}_partial.csv"
+            try:
+                with open(filename, 'w', newline='', encoding='utf-8') as file:
+                    writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                    writer.writeheader()
+                    writer.writerows(self.articles)
+                self.last_save_time = current_time
+                print(f"\nProgress saved to {filename}: {len(self.articles)} articles")
+                self.logger.info(f"Progress saved to {filename}: {len(self.articles)} articles")
+            except Exception as e:
+                self.logger.error(f"Error saving progress: {str(e)}")
+
+    def scrape_topic(self, topic_url, topic):
+        """Incrementally scrape articles while loading more content"""
+        driver = self.create_driver()
+        try:
+            driver.get(topic_url)
+            time.sleep(1)
+
+            while not self.is_interrupted:
+                # Scrape currently visible articles
+                new_articles = self.extract_visible_articles(driver)
+                if new_articles:
+                    # Process new articles in parallel
+                    self.process_articles_batch(new_articles)
+                    self.logger.info(f"Scraped {len(new_articles)} new articles. Total: {len(self.articles)}")
+                    # Save progress after each batch
+                    self.save_progress(topic)
+
+                # Try to load more articles
+                try:
+                    parent_div = driver.find_element(By.CLASS_NAME, "IVNry")
+                    load_more_button = parent_div.find_element(By.TAG_NAME, "button")
+                    self.scroll_to_element(driver, load_more_button)
+                    driver.execute_script("arguments[0].click();", load_more_button)
+                    time.sleep(1)
+                except NoSuchElementException:
+                    self.logger.info("No more articles to load")
+                    break
+                except Exception as e:
+                    self.logger.error(f"Error clicking load more: {str(e)}")
+                    break
+
+            # Save final results
+            if self.articles:
+                return self.save_to_csv(self.articles, topic, final=True)
+            return None
+
+        except Exception as e:
+            self.logger.error(f"Error in scrape_topic: {str(e)}")
+            if self.articles:
+                return self.save_to_csv(self.articles, topic, final=True)
+            return None
+        finally:
+            driver.quit()
+
+    def process_articles_batch(self, article_batch):
+        """Process a batch of articles in parallel"""
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.scrape_article_parallel, article_data)
+                      for article_data in article_batch]
+
+            for future in as_completed(futures):
+                try:
+                    article = future.result()
+                    if article:
+                        self.articles.append(article)
+                        self.logger.info(f"Successfully scraped: {article['title'][:50]}...")
+                except Exception as e:
+                    self.logger.error(f"Error processing article: {str(e)}")
+
+    def scrape_article_parallel(self, article_data):
+        """Scrape a single article with its own driver instance"""
+        driver = self.create_driver()
+        try:
+            url = article_data['link']
+            driver.get(url)
+
+            try:
+                title_element = WebDriverWait(driver, 5).until(
+                    EC.presence_of_element_located((By.TAG_NAME, "h1"))
+                )
+            except TimeoutException:
+                pass
+
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+
+            # Extract title
+            title = None
+            title_element = soup.find('h1')
+            if title_element:
+                title = title_element.get_text().strip()
+
+            if not title:
+                title_elements = soup.find_all(['h1', 'div'], class_=['_23498', 'articletitle', 'headline', 'title'])
+                for element in title_elements:
+                    if element.get_text().strip():
+                        title = element.get_text().strip()
+                        break
+
+            # Extract description
+            desc = None
+            desc_element = soup.find('div', class_='_s30J clearfix')
+            if desc_element:
+                desc = desc_element.get_text().strip("<br>").strip("<!-- -->").strip('<div>')
+
+            if not title:
+                meta_title = soup.find('meta', property='og:title')
+                if meta_title:
+                    title = meta_title.get('content', '').strip()
+
+            return {
+                'title': title or 'Title not found',
+                'desc': desc or 'Description not found',
+                'date': article_data['date'] or 'Date not found',
+                'link': url
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error scraping article {url}: {str(e)}")
+            return None
+        finally:
+            driver.quit()
+
+    def extract_date(self, container):
+        """Extract the date from the article container"""
+        try:
+            date_div = container.find('div', class_='ZxBIG')
+            if date_div:
+                contents = list(date_div.descendants)
+                for content in contents:
+                    if hasattr(content, 'strip') and 'IST' in str(content):
+                        text = str(content).strip()
+                        if '(IST)' in text:
+                            return text.strip()
+            return None
+        except Exception as e:
+            self.logger.error(f"Error extracting date: {str(e)}")
+            return None
+
+    def save_to_csv(self, articles, topic, final=False):
+        """Save articles to CSV file"""
+        if not articles:
+            self.logger.error("No articles to save")
+            return None
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"toi_{topic}articles_{timestamp}_{'final' if final else 'partial'}.csv"
+
+        try:
+            with open(filename, 'w', newline='', encoding='utf-8') as file:
+                writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                writer.writeheader()
+                writer.writerows(articles)
+                self.logger.info(f"Successfully saved {len(articles)} articles to: {filename}")
+            return filename
+        except Exception as e:
+            self.logger.error(f"Error saving to CSV: {str(e)}")
+            return None
+
+def parse_arguments():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description='Scrape articles from Times of India by topic')
+    
+    parser.add_argument('topic', type=str, 
+                        help='Topic to scrape (e.g., "RSS", "Covid", "India")')
+    
+    parser.add_argument('-w', '--workers', type=int, default=4,
+                        help='Number of worker threads (default: 4)')
+    
+    parser.add_argument('-i', '--interval', type=int, default=300,
+                        help='Auto-save interval in seconds (default: 300)')
+    
+    return parser.parse_args()
+
+def main():
+    try:
+        # Parse command line arguments
+        args = parse_arguments()
+        
+        # Initialize the scraper with command line arguments
+        scraper = TOIArticleScraper(max_workers=args.workers)
+        scraper.save_interval = args.interval
+        
+        # Get topic from command line argument
+        topic = args.topic
+        topic_url = f"{scraper.base_url}/topic/{topic}/news"
+
+        print(f"\nScraping {topic}-related articles from Times of India...")
+        print("Press Ctrl+C at any time to save progress and exit.")
+
+        final_csv = scraper.scrape_topic(topic_url, topic)
+
+        if final_csv:
+            print(f"\nArticles have been saved to: {final_csv}")
+            print(f"Total articles scraped: {len(scraper.articles)}")
+        else:
+            print("\nError saving to final CSV file")
+
+    except KeyboardInterrupt:
+        print("\nProcess interrupted by user. Saving progress...")
+        if scraper.articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress and exiting.")
+    except Exception as e:
+        print(f"\nAn error occurred: {str(e)}")
+        if 'scraper' in locals() and scraper.articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress despite error.")
+
+if __name__ == "__main__":
+    main()

src/scrapers/wion_scraper.py

@@ -0,0 +1,379 @@
+import os
+import time
+import logging
+import csv
+import signal
+import sys
+import argparse
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import (
+    TimeoutException, NoSuchElementException, ElementClickInterceptedException,
+    StaleElementReferenceException, WebDriverException
+)
+from bs4 import BeautifulSoup
+from urllib3.exceptions import ProtocolError
+
+from utils.webdriver_utils import create_chrome_driver, scroll_to_element
+
+class WIONArticleScraper:
+    def __init__(self, max_workers=4, articles_per_page=10, max_retries=3):
+        self.max_workers = max_workers
+        self.articles_per_page = articles_per_page
+        self.max_retries = max_retries
+        self.base_url = "https://www.wionews.com/search"
+        self.fetched_articles = set()
+        self.articles = []
+        self.is_interrupted = False
+        self.last_save_time = time.time()
+        self.save_interval = 300  # Save every 5 minutes
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger(__name__)
+
+        signal.signal(signal.SIGINT, self.signal_handler)
+        signal.signal(signal.SIGTERM, self.signal_handler)
+
+    def signal_handler(self, signum, frame):
+        print("\nReceived interrupt signal. Saving progress and shutting down...")
+        self.is_interrupted = True
+        if self.articles:
+            self.save_progress("interrupted", force=True)
+        sys.exit(0)
+
+    # def create_driver(self):
+    #     chrome_options = webdriver.ChromeOptions()
+    #     chrome_options.add_argument('--headless')
+    #     chrome_options.add_argument('--no-sandbox')
+    #     chrome_options.add_argument('--disable-dev-shm-usage')
+    #     chrome_options.add_argument('--disable-extensions')
+    #     chrome_options.add_argument('--disable-gpu')
+    #     chrome_options.add_argument('--disable-infobars')
+    #     chrome_options.add_argument('--disable-notifications')
+    #     chrome_options.add_argument('--blink-settings=imagesEnabled=false')  # Disable images
+    #     chrome_options.page_load_strategy = 'eager'
+
+    #     # Add performance preferences
+    #     chrome_options.add_experimental_option('prefs', {
+    #         'profile.default_content_setting_values.notifications': 2,
+    #         'profile.managed_default_content_settings.images': 2,
+    #         'disk-cache-size': 4096
+    #     })
+
+    #     return webdriver.Chrome(options=chrome_options)
+
+    def create_driver(self):
+        """Create and return a new Chrome driver instance"""
+        return create_chrome_driver(headless=True, load_images=False)
+
+    def wait_for_page_load(self, driver, url, timeout=10, retries=3):
+        """Wait for page load with retries"""
+        for attempt in range(retries):
+            try:
+                driver.set_page_load_timeout(timeout)
+                driver.get(url)
+
+                # Wait for DOM to be ready
+                WebDriverWait(driver, timeout).until(
+                    lambda d: d.execute_script('return document.readyState') == 'complete'
+                )
+
+                return True
+            except (TimeoutException, WebDriverException, ProtocolError) as e:
+                if attempt == retries - 1:
+                    self.logger.warning(f"Failed to load {url} after {retries} attempts: {str(e)}")
+                    return False
+                else:
+                    self.logger.info(f"Retrying page load for {url} (attempt {attempt + 2}/{retries})")
+                    time.sleep(2 * (attempt + 1))  # Exponential backoff
+                    continue
+            except Exception as e:
+                self.logger.error(f"Unexpected error loading {url}: {str(e)}")
+                return False
+        return False
+
+    def get_total_pages(self, driver, search_term):
+        try:
+            if not self.wait_for_page_load(driver, f"{self.base_url}?page=1&title={search_term}"):
+                return 20
+
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+            pagination = soup.find('ul', class_='pagination')
+            if pagination:
+                pages = pagination.find_all('li')
+                if pages:
+                    last_page = pages[-2].text.strip()
+                    return int(last_page)
+            return 20
+        except Exception as e:
+            self.logger.error(f"Error getting total pages: {str(e)}")
+            return 20
+
+    def extract_visible_articles(self, driver):
+        """Extract currently visible articles from the page"""
+        try:
+            # Wait for article containers to be present
+            WebDriverWait(driver, 10).until(
+                EC.presence_of_element_located((By.CLASS_NAME, 'gh-archive-page-post-content-main'))
+            )
+
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+            article_containers = soup.find_all('div', class_='gh-archive-page-post-content-main')
+
+            new_articles = []
+            for container in article_containers:
+                try:
+                    link_element = container.find('a', href=True)
+                    date = self.extract_date(container)
+
+                    if link_element and link_element['href']:
+                        full_link = link_element['href'] if link_element['href'].startswith('http') else 'https://www.wionews.com' + link_element['href']
+                        if full_link not in self.fetched_articles:
+                            self.fetched_articles.add(full_link)
+                            new_articles.append({'link': full_link, 'date': date})
+                except Exception as e:
+                    self.logger.error(f"Error processing article container: {str(e)}")
+                    continue
+
+            return new_articles
+        except Exception as e:
+            self.logger.error(f"Error extracting visible articles: {str(e)}")
+            return []
+
+    def extract_date(self, container):
+        try:
+            time_element = container.find('time', class_='gh-post-info__date')
+
+            if time_element:
+                date = time_element.get('datetime', time_element.get_text().strip())
+                return date
+
+            post_info = container.find('div', class_='gh-post-info')
+            if post_info:
+                time_element = post_info.find('time')
+                if time_element:
+                    date = time_element.get('datetime', time_element.get_text().strip())
+                    return date
+
+            return None
+        except Exception as e:
+            self.logger.error(f"Error extracting date: {str(e)}")
+            return None
+
+    def scrape_topic(self, search_term, topic):
+        driver = self.create_driver()
+        try:
+            total_pages = self.get_total_pages(driver, search_term)
+            total_expected_articles = total_pages * self.articles_per_page
+            self.logger.info(f"Found {total_pages} pages to scrape (approximately {total_expected_articles} articles)")
+
+            for page in range(1, total_pages + 1):
+                if self.is_interrupted:
+                    break
+
+                page_url = f"{self.base_url}?page={page}&title={search_term}"
+                articles_scraped = len(self.articles)
+                progress_percentage = (articles_scraped / total_expected_articles) * 100
+
+                self.logger.info(f"Scraping page {page}/{total_pages} - Progress: {articles_scraped}/{total_expected_articles} articles ({progress_percentage:.1f}%)")
+
+                # Try loading the page with retries
+                if not self.wait_for_page_load(driver, page_url):
+                    self.logger.warning(f"Skipping page {page} due to load failure")
+                    continue
+
+                new_articles = self.extract_visible_articles(driver)
+                if new_articles:
+                    self.process_articles_batch(new_articles)
+                    self.logger.info(f"Scraped {len(new_articles)}/{self.articles_per_page} articles from page {page}")
+                    self.save_progress(topic)
+                else:
+                    self.logger.warning(f"No articles found on page {page}")
+
+            if self.articles:
+                return self.save_to_csv(self.articles, topic, final=True)
+            return None
+
+        except Exception as e:
+            self.logger.error(f"Error in scrape_topic: {str(e)}")
+            if self.articles:
+                return self.save_to_csv(self.articles, topic, final=True)
+            return None
+        finally:
+            driver.quit()
+
+    def scrape_article_parallel(self, article_data):
+        driver = self.create_driver()
+        url = article_data['link']
+
+        for attempt in range(self.max_retries):
+            try:
+                if not self.wait_for_page_load(driver, url):
+                    if attempt == self.max_retries - 1:
+                        raise TimeoutException(f"Failed to load article after {self.max_retries} attempts")
+                    continue
+
+                # Wait for main content to be present
+                WebDriverWait(driver, 10).until(
+                    EC.presence_of_element_located((By.TAG_NAME, "article"))
+                )
+
+                soup = BeautifulSoup(driver.page_source, 'html.parser')
+
+                # Extract title with multiple fallbacks
+                title = None
+                for selector in ['h1.article-main-title', 'h1.headline', 'h1']:
+                    title_element = soup.select_one(selector)
+                    if title_element:
+                        title = title_element.get_text().strip()
+                        break
+
+                # Extract description with multiple fallbacks
+                desc = None
+                for class_name in ['post-content', 'article-main-content', 'post-page-main-content']:
+                    content_div = soup.find('div', class_=class_name)
+                    if content_div:
+                        p_tags = content_div.find_all('p')
+                        if p_tags:
+                            desc = ' '.join([p.get_text().strip() for p in p_tags])
+                            break
+
+                return {
+                    'title': title or 'Title not found',
+                    'desc': desc or 'Description not found',
+                    'date': article_data['date'] or 'Date not found',
+                    'link': url
+                }
+
+            except Exception as e:
+                if attempt == self.max_retries - 1:
+                    self.logger.error(f"Error scraping article {url}: {str(e)}")
+                    return None
+                time.sleep(2 * (attempt + 1))  # Exponential backoff
+            finally:
+                if attempt == self.max_retries - 1:
+                    driver.quit()
+
+    def process_articles_batch(self, article_batch):
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.scrape_article_parallel, article_data)
+                      for article_data in article_batch]
+
+            successful_articles = 0
+            for future in as_completed(futures):
+                try:
+                    article = future.result()
+                    if article:
+                        self.articles.append(article)
+                        successful_articles += 1
+                        self.logger.info(f"Successfully scraped: {article['title'][:50]}...")
+                except Exception as e:
+                    self.logger.error(f"Error processing article: {str(e)}")
+
+            if successful_articles < len(article_batch):
+                self.logger.warning(f"Only processed {successful_articles}/{len(article_batch)} articles in this batch")
+
+    def save_progress(self, topic, force=False):
+        current_time = time.time()
+        if force or (current_time - self.last_save_time >= self.save_interval and self.articles):
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"wion_{topic}articles_{timestamp}_partial.csv"
+            try:
+                with open(filename, 'w', newline='', encoding='utf-8') as file:
+                    writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                    writer.writeheader()
+                    writer.writerows(self.articles)
+                self.last_save_time = current_time
+                print(f"\nProgress saved to {filename}: {len(self.articles)} articles")
+                self.logger.info(f"Progress saved to {filename}: {len(self.articles)} articles")
+            except Exception as e:
+                self.logger.error(f"Error saving progress: {str(e)}")
+
+    def save_to_csv(self, articles, topic, final=False):
+        if not articles:
+            self.logger.error("No articles to save")
+            return None
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"wion_{topic}articles_{timestamp}_{'final' if final else 'partial'}.csv"
+
+        try:
+            with open(filename, 'w', newline='', encoding='utf-8') as file:
+                writer = csv.DictWriter(file, fieldnames=['title', 'desc', 'date', 'link'])
+                writer.writeheader()
+                writer.writerows(articles)
+                self.logger.info(f"Successfully saved {len(articles)} articles to: {filename}")
+            return filename
+        except Exception as e:
+            self.logger.error(f"Error saving to CSV: {str(e)}")
+            return None
+
+def parse_arguments():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description='Scrape articles from WION News by topic')
+    
+    parser.add_argument('topic', type=str, 
+                        help='Topic to scrape (e.g., "RSS", "Covid", "India")')
+    
+    parser.add_argument('-w', '--workers', type=int, default=4,
+                        help='Number of worker threads (default: 4)')
+    
+    parser.add_argument('-i', '--interval', type=int, default=300,
+                        help='Auto-save interval in seconds (default: 300)')
+                        
+    parser.add_argument('-a', '--articles-per-page', type=int, default=10,
+                        help='Expected number of articles per page (default: 10)')
+                        
+    parser.add_argument('-r', '--retries', type=int, default=3,
+                        help='Maximum retries for failed requests (default: 3)')
+    
+    return parser.parse_args()
+
+def main():
+    try:
+        # Parse command line arguments
+        args = parse_arguments()
+        
+        # Initialize the scraper with command line arguments
+        scraper = WIONArticleScraper(
+            max_workers=args.workers, 
+            articles_per_page=args.articles_per_page,
+            max_retries=args.retries
+        )
+        scraper.save_interval = args.interval
+        
+        # Get topic from command line argument
+        topic = args.topic
+        
+        print(f"\nScraping {topic}-related articles from WION News...")
+        print("Press Ctrl+C at any time to save progress and exit.")
+
+        final_csv = scraper.scrape_topic(topic.lower(), topic)
+
+        if final_csv:
+            print(f"\nArticles have been saved to: {final_csv}")
+            print(f"Total articles scraped: {len(scraper.articles)}")
+        else:
+            print("\nError saving to final CSV file")
+
+    except KeyboardInterrupt:
+        print("\nProcess interrupted by user. Saving progress...")
+        if scraper.articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress and exiting.")
+    except Exception as e:
+        print(f"\nAn error occurred: {str(e)}")
+        if 'scraper' in locals() and scraper.articles:
+            scraper.save_progress(topic, force=True)
+        print("Saved progress despite error.")
+
+if __name__ == "__main__":
+    main()

src/utils/webdriver_utils.py

@@ -0,0 +1,145 @@
+"""
+Utilities for creating and managing Selenium WebDriver instances.
+This module provides reusable functions for browser automation.
+"""
+
+import time
+import logging
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from urllib3.exceptions import ProtocolError
+
+logger = logging.getLogger(__name__)
+
+def create_chrome_driver(headless=True, load_images=False, page_load_strategy='eager'):
+    """
+    Create and configure a Chrome WebDriver instance with optimized settings.
+    
+    Args:
+        headless (bool): Whether to run Chrome in headless mode
+        load_images (bool): Whether to load images
+        page_load_strategy (str): Page load strategy ('normal', 'eager', or 'none')
+        
+    Returns:
+        webdriver.Chrome: Configured Chrome WebDriver instance
+    """
+    chrome_options = webdriver.ChromeOptions()
+    
+    if headless:
+        chrome_options.add_argument('--headless')
+    
+    # Common performance optimizations
+    chrome_options.add_argument('--no-sandbox')
+    chrome_options.add_argument('--disable-dev-shm-usage')
+    chrome_options.add_argument('--disable-extensions')
+    chrome_options.add_argument('--disable-gpu')
+    chrome_options.add_argument('--disable-infobars')
+    chrome_options.add_argument('--disable-notifications')
+    
+    if not load_images:
+        chrome_options.add_argument('--blink-settings=imagesEnabled=false')
+    
+    chrome_options.page_load_strategy = page_load_strategy
+    
+    # Performance preferences
+    chrome_options.add_experimental_option('prefs', {
+        'profile.default_content_setting_values.notifications': 2,
+        'profile.managed_default_content_settings.images': 2 if not load_images else 0,
+        'disk-cache-size': 4096
+    })
+    
+    return webdriver.Chrome(options=chrome_options)
+
+def wait_for_page_load(driver, url, timeout=10, retries=3, backoff_factor=2):
+    """
+    Load a URL with retries and exponential backoff.
+    
+    Args:
+        driver (webdriver.Chrome): WebDriver instance
+        url (str): URL to load
+        timeout (int): Page load timeout in seconds
+        retries (int): Number of retry attempts
+        backoff_factor (int): Factor to multiply wait time by on each retry
+        
+    Returns:
+        bool: Whether page load was successful
+    """
+    for attempt in range(retries):
+        try:
+            driver.set_page_load_timeout(timeout)
+            driver.get(url)
+            
+            # Wait for DOM to be ready
+            WebDriverWait(driver, timeout).until(
+                lambda d: d.execute_script('return document.readyState') == 'complete'
+            )
+            
+            return True
+            
+        except (TimeoutException, WebDriverException, ProtocolError) as e:
+            if attempt == retries - 1:
+                logger.warning(f"Failed to load {url} after {retries} attempts: {str(e)}")
+                return False
+            else:
+                wait_time = backoff_factor * (attempt + 1)
+                logger.info(f"Retrying page load for {url} (attempt {attempt + 2}/{retries}) in {wait_time}s")
+                time.sleep(wait_time)
+                continue
+                
+        except Exception as e:
+            logger.error(f"Unexpected error loading {url}: {str(e)}")
+            return False
+            
+    return False
+
+def scroll_to_element(driver, element):
+    """
+    Scroll the page to make an element visible.
+    
+    Args:
+        driver (webdriver.Chrome): WebDriver instance
+        element: WebElement to scroll to
+    """
+    try:
+        driver.execute_script("arguments[0].scrollIntoView(true);", element)
+        driver.execute_script("window.scrollBy(0, -100);")  # Adjust to avoid navbar overlay
+    except Exception as e:
+        logger.error(f"Error scrolling to element: {str(e)}")
+
+def scroll_to_bottom(driver, scroll_pause_time=1.0, num_scrolls=None):
+    """
+    Scroll to the bottom of the page incrementally.
+    
+    Args:
+        driver (webdriver.Chrome): WebDriver instance
+        scroll_pause_time (float): Time to pause between scrolls
+        num_scrolls (int, optional): Maximum number of scrolls to perform
+    """
+    # Get scroll height
+    last_height = driver.execute_script("return document.body.scrollHeight")
+    
+    scrolls_performed = 0
+    
+    while True:
+        # Check if we've reached the scroll limit
+        if num_scrolls is not None and scrolls_performed >= num_scrolls:
+            break
+            
+        # Scroll down to bottom
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+        
+        # Wait to load page
+        time.sleep(scroll_pause_time)
+        
+        # Calculate new scroll height and compare with last scroll height
+        new_height = driver.execute_script("return document.body.scrollHeight")
+        if new_height == last_height:
+            break
+            
+        last_height = new_height
+        scrolls_performed += 1
+        
+    return scrolls_performed