Spaces:
Sleeping
Sleeping
| import time | |
| import numpy as np | |
| import pandas as pd | |
| import requests | |
| import streamlit as st | |
| from bs4 import BeautifulSoup | |
| from sentence_transformers import SentenceTransformer, util | |
| def preprocess(): | |
| # Base URL for navigation | |
| base_url = 'https://courses.analyticsvidhya.com/collections/courses?page=' | |
| course_list_url = "https://courses.analyticsvidhya.com/" | |
| # List to hold course data | |
| courses = [] | |
| page_number = 1 # Start with the first page | |
| while True: | |
| # Construct URL for the current page | |
| current_page_url = base_url + str(page_number) | |
| print(f"Processing page {page_number}...") | |
| # Get the current page content | |
| response = requests.get(current_page_url) | |
| if response.status_code != 200: | |
| print(f"Failed to fetch page {page_number}. Status code: {response.status_code}") | |
| break | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| # Find all course cards | |
| course_cards = soup.find_all('li', class_='products__list-item') | |
| if not course_cards: | |
| print("No more courses found. Ending extraction.") | |
| break | |
| # Extract course data from each card | |
| for course_card in course_cards: | |
| title_tag = course_card.find('h3') | |
| link_tag = course_card.find('a') | |
| if title_tag and link_tag: # Check if both title and link exist | |
| title = title_tag.text.strip() | |
| course_link = link_tag['href'] | |
| # Construct full course URL (assume relative links) | |
| course_url = course_list_url.rstrip('/') + course_link | |
| # Visit each course link to get the description | |
| course_response = requests.get(course_url) | |
| if course_response.status_code == 200: | |
| course_soup = BeautifulSoup(course_response.content, 'html.parser') | |
| description_tag = course_soup.find('div', class_='fr-view') # Adjust based on actual class or tag | |
| description = description_tag.text.strip() if description_tag else 'No description available' | |
| curriculum_tag = course_soup.find('ul', class_='course-curriculum__chapter-content') # Adjust based on actual class or tag | |
| curriculum = curriculum_tag.text.strip() if curriculum_tag else 'No curriculum available' | |
| #enroll_tag = course_soup.find('article', class_='section__content section__content___ae733') # Adjust based on actual class or tag | |
| #enroll = enroll_tag.text.strip() if enroll_tag else 'No enroll available' | |
| instructor_tag = course_soup.find('section', class_='text-image section-height__medium section__content-alignment--left text-image___07200') # Adjust based on actual class or tag | |
| instructor = instructor_tag.text.strip() if instructor_tag else 'No instructor available' | |
| # Append the data to the list | |
| courses.append({'title': title, 'description': description, 'Course curriculum': curriculum, 'About the Instructor': instructor}) | |
| else: | |
| print(f"Failed to fetch course page: {course_url}") | |
| # Sleep to avoid overwhelming the server (optional) | |
| time.sleep(1) | |
| else: | |
| print("Skipped a course card due to missing title or link.") | |
| # Move to the next page | |
| page_number += 1 | |
| # break | |
| # Save the collected data to a CSV file | |
| df = pd.DataFrame(courses) | |
| df.to_csv('courses.csv', index=False) | |
| print("Data collection complete. Saved to courses.csv.") | |
| # Load the data | |
| df = pd.read_csv('courses.csv') | |
| # Combine relevant text fields for embedding (e.g., title, description, curriculum) | |
| df['combined_text'] = df['title'] + ' ' + df['description'] + ' ' + df['Course curriculum'] + ' ' + df['About the Instructor'] | |
| # Load a pre-trained model for embeddings | |
| model = SentenceTransformer('all-MiniLM-L6-v2') | |
| # Create embeddings for each course | |
| embeddings = model.encode(df['combined_text'].tolist(), convert_to_tensor=True) | |
| # Save embeddings and DataFrame for later use | |
| np.save('course_embeddings.npy', embeddings) | |
| df.to_csv('courses_with_embeddings.csv', index=False) | |
| # Load embeddings and DataFrame | |
| embeddings = np.load('course_embeddings.npy') | |
| df = pd.read_csv('courses_with_embeddings.csv') |