scrapper.py

import requests
from bs4 import BeautifulSoup
import csv
import os

def extract_course_info(html_content, url):
    soup = BeautifulSoup(html_content, 'html.parser')
    
    # Extract course name
    course_name = soup.title.string if soup.title else "Course name not found"
    
    # Extract key takeaways - Updated selector to match the provided HTML
    key_takeaways = []
    checklist_container = soup.find('div', class_='checklist__container')
    if checklist_container:
        takeaway_items = checklist_container.find_all('li', class_='checklist__list-item')
        for item in takeaway_items:
            p_tag = item.find('p')
            if p_tag:
                # Remove the icon and get clean text
                takeaway_text = p_tag.text.replace('\uf00c', '').strip()
                # Remove "fa fa-check" text if present
                takeaway_text = takeaway_text.replace('fa fa-check', '').strip()
                key_takeaways.append(takeaway_text)
    
    # Extract course time, ratings, and difficulty level - FIXED PART
    # Use safer method to handle NoneType and avoid errors
    course_time = soup.find('li', class_='text-icon__list-item')
    course_time_text = course_time.find('h4').text if course_time else "Course time not found"
    
    ratings = course_time.find_next_sibling('li').find('h4').text if course_time and course_time.find_next_sibling('li') else "Ratings not found"
    
    difficulty_sibling = course_time.find_next_sibling('li').find_next_sibling('li') if course_time and course_time.find_next_sibling('li') else None
    difficulty = difficulty_sibling.find('h4').text if difficulty_sibling else "Difficulty level not found"

    # Extract course description
    description = "Description not found"
    description_section = soup.find('div', class_='course-description')
    if description_section:
        first_p = description_section.find('p')
        if first_p:
            description = first_p.text.strip()
    
    return {
        "course_name": course_name,
        "key_takeaways": ', '.join(key_takeaways) if key_takeaways else "No key takeaways found",
        "course_time": course_time_text,
        "ratings": ratings,
        "difficulty": difficulty,
        "description": description,
        "website": url
    }

def append_to_csv(course_info, csv_filename="course_data.csv"):
    file_exists = os.path.isfile(csv_filename)
    
    with open(csv_filename, mode='a', newline='', encoding='utf-8') as file:
        writer = csv.writer(file)
        
        if not file_exists:
            writer.writerow(["Course Name", "Key Takeaways", "Course Time", "Ratings", "Difficulty", "Description", "Website"])
        
        writer.writerow([
            course_info["course_name"],
            course_info["key_takeaways"],
            course_info["course_time"],
            course_info["ratings"],
            course_info["difficulty"],
            course_info["description"],
            course_info["website"]
        ])

# Add headers to help avoid blocking
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}

# Updated list of URLs to process
urls = [
    "https://courses.analyticsvidhya.com/courses/genai-applied-to-quantitative-finance-for-control-implementation",
    "https://courses.analyticsvidhya.com/courses/navigating-llm-tradeoffs-techniques-for-speed-cost-scale-and-accuracy",
    "https://courses.analyticsvidhya.com/courses/creating-problem-solving-agents-using-genai-for-action-composition",
    "https://courses.analyticsvidhya.com/courses/improving-real-world-rag-systems-key-challenges",
    "https://courses.analyticsvidhya.com/courses/choosing-the-right-LLM-for-your-business",
    "https://courses.analyticsvidhya.com/courses/building-smarter-llms-with-mamba-and-state-space-model",
    "https://courses.analyticsvidhya.com/courses/genai-a-way-of-life",
    "https://courses.analyticsvidhya.com/courses/building-llm-applications-using-prompt-engineering-free",
    "https://courses.analyticsvidhya.com/courses/building-your-first-computer-vision-model",
    
    # New URLs
    "https://courses.analyticsvidhya.com/courses/bagging-boosting-ML-Algorithms",
    "https://courses.analyticsvidhya.com/courses/midjourney_from_inspiration_to_implementation",
    "https://courses.analyticsvidhya.com/courses/free-understanding-linear-regression",
    "https://courses.analyticsvidhya.com/courses/The%20Working%20of%20Neural%20Networks",
    "https://courses.analyticsvidhya.com/courses/free-unsupervised-ml-guide",
    "https://courses.analyticsvidhya.com/courses/building-first-rag-systems-using-llamaindex",
    "https://courses.analyticsvidhya.com/courses/data-preprocessing",
    "https://courses.analyticsvidhya.com/courses/exploring-stability-ai",
    "https://courses.analyticsvidhya.com/courses/free-building-textclassification-natural-language-processing",
    
    "https://courses.analyticsvidhya.com/courses/getting-started-with-llms",
    "https://courses.analyticsvidhya.com/courses/introduction-to-generative-ai",
    "https://courses.analyticsvidhya.com/courses/nano-course-dreambooth-stable-diffusion-for-custom-images",
    "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-for-deep-learning-in-2023",
    "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-to-become-a-data-scientist-in-twenty-twenty-four",
    "https://courses.analyticsvidhya.com/courses/building-large-language-models-for-code",
    "https://courses.analyticsvidhya.com//bundles/certified-ai-ml-blackbelt-plus",
    "https://courses.analyticsvidhya.com/courses/machine-learning-summer-training",
    "https://courses.analyticsvidhya.com/courses/ai-ethics-fractal",
    
    "https://courses.analyticsvidhya.com/courses/a-comprehensive-learning-path-to-become-a-data-engineer-in-2022",
    "https://courses.analyticsvidhya.com/bundles/certified-business-analytics-program",
    "https://courses.analyticsvidhya.com/bundles/certified-machine-learning-master-s-program-mlmp",
    "https://courses.analyticsvidhya.com/bundles/certified-natural-language-processing-master-s-program",
    "https://courses.analyticsvidhya.com/bundles/certified-computer-vision-masters-program",
    "https://courses.analyticsvidhya.com/courses/applied-machine-learning-beginner-to-professional",
    "https://courses.analyticsvidhya.com/courses/ace-data-science-interviews",
    "https://courses.analyticsvidhya.com/courses/writing-powerful-data-science-articles",
    "https://courses.analyticsvidhya.com/courses/machine-learning-certification-course-for-beginners",
    
    "https://courses.analyticsvidhya.com/courses/data-science-career-conclave",
    "https://courses.analyticsvidhya.com/courses/top-data-science-projects-for-analysts-and-data-scientists",
    "https://courses.analyticsvidhya.com/courses/getting-started-with-git-and-github-for-data-science-professionals",
    "https://courses.analyticsvidhya.com/courses/machine-learning-starter-program",
    "https://courses.analyticsvidhya.com/courses/data-science-hacks-tips-and-tricks",
    "https://courses.analyticsvidhya.com/courses/introduction-to-analytics",
    "https://courses.analyticsvidhya.com/courses/introduction-to-pytorch-for-deeplearning",
    "https://courses.analyticsvidhya.com/courses/introductory-data-science-for-business-managers",
    "https://courses.analyticsvidhya.com/courses/intro-to-nlp",
    
    "https://courses.analyticsvidhya.com/courses/getting-started-with-decision-trees",
    "https://courses.analyticsvidhya.com/courses/introduction-to-data-science",
    "https://courses.analyticsvidhya.com/courses/loan-prediction-practice-problem-using-python",
    "https://courses.analyticsvidhya.com/courses/big-mart-sales-prediction-using-r",
    "https://courses.analyticsvidhya.com/courses/twitter-sentiment-analysis",
    "https://courses.analyticsvidhya.com/courses/pandas-for-data-analysis-in-python",
    "https://courses.analyticsvidhya.com/courses/support-vector-machine-svm-in-python-and-r",
    "https://courses.analyticsvidhya.com/courses/evaluation-metrics-for-machine-learning-models"
]

# Process each URL
for url in urls:
    try:
        print(f"Processing {url}...")
        response = requests.get(url, headers=headers)
        response.raise_for_status()  # Check for HTTP errors

        html = response.content
        course_info = extract_course_info(html, url)
        append_to_csv(course_info)
        
        print(f"Data for {url} has been successfully appended.")
    except Exception as e:
        print(f"Failed to process {url}: {e}")