reto.py

import spacy
import wikipedia
import requests
import re
import wikipediaapi
import textblob
from textblob import download_corpora
import random
from textblob import TextBlob
from urllib.parse import urljoin, urlparse
import time

download_corpora.download_all()

# Load spaCy model for better sentence processing
try:
    nlp = spacy.load("en_core_web_sm")
except:
    print("spaCy model not found. Please install it with: python -m spacy download en_core_web_sm")
    nlp = None

# Option 1: Using wikipedia-api library (recommended)
def get_article_info_wiki_api(article_name):
    """Get article content, categories, and links using wikipedia-api library"""
    try:
        wiki_wiki = wikipediaapi.Wikipedia(
            'en',
            extract_format=wikipediaapi.ExtractFormat.WIKI
        )
        page = wiki_wiki.page(article_name)
        
        if not page.exists():
            return None, None, None
        
        # Get categories and remove 'Category:' prefix
        categories = [cat.replace('Category:', '') for cat in page.categories.keys()]
        
        # Get external links from the page
        external_links = extract_external_links(page.fullurl)
        
        return page.text, categories if categories else ['Uncategorized'], external_links
    except Exception as e:
        print(f"Error in get_article_info_wiki_api: {e}")
        return None, None, None

# Option 2: Using requests and regex (updated approach)
def get_article_info(article_name):
    """Get article content and categories using web scraping approach"""
    try:
        # Get page content using wikipedia library with better error handling
        try:
            page = wikipedia.page(article_name, auto_suggest=True)
            summary = page.summary
            content = page.content
            page_url = page.url
        except wikipedia.exceptions.DisambiguationError as e:
            # If it's a disambiguation page, use the first suggestion
            print(f"Disambiguation page. Using first option: {e.options[0]}")
            page = wikipedia.page(e.options[0], auto_suggest=False)
            summary = page.summary
            content = page.content
            page_url = page.url
        except wikipedia.exceptions.PageError:
            print(f"Page '{article_name}' not found on Wikipedia")
            return None, None, None
        
        # Get categories via web scraping with updated regex
        try:
            r = requests.get(page_url, timeout=10)
            html = r.text
            
            # Updated regex pattern for categories
            catlinks_regexp = re.compile(r'<div class="mw-normal-catlinks".*?>(.*?)<\/div>', re.DOTALL)
            catnames_regexp = re.compile(r'<a[^>]*>([^<]*)<\/a>')
            
            cat_src = catlinks_regexp.findall(html)
            if not cat_src:
                # Try alternative pattern
                catlinks_regexp = re.compile(r'<div id="catlinks".*?>(.*?)<\/div>', re.DOTALL)
                cat_src = catlinks_regexp.findall(html)
            
            if not cat_src:
                categories = ['Uncategorized']
            else:
                cats = catnames_regexp.findall(cat_src[0])
                # Skip the first element which is typically "Categories:"
                categories = cats[1:] if len(cats) > 1 else ['Uncategorized']
                
            # Get external links
            external_links = extract_external_links(page_url)
            
            return content, categories, external_links
            
        except requests.RequestException as e:
            print(f"Request error: {e}")
            # Fallback to using wikipedia library categories if available
            if hasattr(page, 'categories'):
                categories = list(page.categories)
                return content, categories if categories else ['Uncategorized'], []
            return content, ['Uncategorized'], []
        
    except Exception as e:
        print(f"Error in get_article_info: {e}")
        return None, None, None

def extract_external_links(wikipedia_url):
    """Extract external links from a Wikipedia page"""
    try:
        response = requests.get(wikipedia_url, timeout=10)
        html_content = response.text
        
        # Find the External links section
        external_links_section = re.search(
            r'<span class="mw-headline" id="External_links">External links</span>.*?(<ul>.*?</ul>)',
            html_content, 
            re.DOTALL
        )
        
        if not external_links_section:
            # Try alternative pattern
            external_links_section = re.search(
                r'<h2><span class="mw-headline" id="External_links">External links</span>.*?(<ul>.*?</ul>)',
                html_content, 
                re.DOTALL
            )
        
        external_links = []
        if external_links_section:
            # Extract links from the section
            links = re.findall(r'<a[^>]*href="([^"]*)"[^>]*>', external_links_section.group(1))
            
            # Filter and format external links
            for link in links:
                # Skip internal Wikipedia links
                if not link.startswith('/wiki/') and not link.startswith('#'):
                    # Make sure it's a valid URL
                    parsed = urlparse(link)
                    if parsed.scheme and parsed.netloc:
                        external_links.append(link)
        
        return external_links[:10]  # Return first 10 external links
        
    except Exception as e:
        print(f"Error extracting external links: {e}")
        return []

def create_sentences_from_categories(categories):
    """Create meaningful sentences from categories"""
    sentences = []
    
    if categories:
        # Create a sentence listing the main categories
        if len(categories) > 3:
            main_categories = random.sample(categories, 3)
            category_sentence = f"This article is primarily about {', '.join(main_categories[:-1])} and {main_categories[-1]}."
        else:
            if len(categories) > 1:
                category_sentence = f"This article is about {', '.join(categories[:-1])} and {categories[-1]}."
            else:
                category_sentence = f"This article is about {categories[0]}."
        
        sentences.append(category_sentence)
        
        # Create additional sentences based on categories
        for category in categories[:5]:  # Limit to first 5 categories
            sentences.append(f"It provides information related to {category}.")
    
    return sentences

def extract_key_sentences(text, num_sentences=3):
    """Extract key sentences from the article text"""
    sentences = []
    
    if text:
        # Use spaCy for better sentence segmentation if available
        if nlp:
            doc = nlp(text)
            sentences = [sent.text for sent in doc.sents]
        else:
            # Fallback to TextBlob
            blob = TextBlob(text)
            sentences = blob.sentences
        
        # Return the first few sentences (usually the most important)
        return sentences[:num_sentences]
    
    return []

def get_references_from_text(text):
    """Extract potential references from text using simple pattern matching"""
    # Look for common citation patterns
    patterns = [
        r'\b(?:https?://|www\.)\S+',
        r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',  # Emails
        r'\b\d{4}\b.*?\b(?:press|university|institute|journal|research|general|code|greeting)\b',  # Year + org
    ]
    
    references = []
    for pattern in patterns:
        matches = re.findall(pattern, text, re.IGNORECASE)
        references.extend(matches)
    
    return references[:5]  # Return first 5 references

# Main execution
userinput = input("Enter a prompt: ")

# Using the wikipedia-api approach (recommended)
print("Using wikipedia-api approach:")
article_text, categories, external_links = get_article_info_wiki_api(userinput)

if article_text and categories:
    print(f"Number of categories: {len(categories)}")
    print(f"Categories: {categories}")
    
    # Create sentences from categories
    category_sentences = create_sentences_from_categories(categories)
    for sentence in category_sentences:
        print(f"- {sentence}")
    
    # Extract key sentences from the article
    key_sentences = extract_key_sentences(article_text)
    for i, sentence in enumerate(key_sentences, 1):
        print(f"{i}. {sentence}")
    
    # Show external links for more data sources
    if external_links:
        print("\nExternal links for more data:")
        for i, link in enumerate(external_links, 1):
            print(f"{i}. {link}")
    else:
        print("\nNo external links found in this article.")
    
    # Extract potential references from text
    references = get_references_from_text(article_text)
    if references:
        print("\nPotential references found in text:")
        for i, ref in enumerate(references, 1):
            print(f"{i}. {ref}")
    
    # Combine categories and article content
    combined_text = " ".join(categories) + " " + article_text[:500]  # First 500 chars of article
    blob = TextBlob(combined_text)
    words = blob.words
    print(f"\nExtracted words from combined content: {set(words[:20])}")  # Show first 20 unique words
    
else:
    print("Page not found using wikipedia-api")

print("\n" + "="*50 + "\n")

# Using the web scraping approach
print("Using web scraping approach:")
article_text, categories, external_links = get_article_info(userinput)

if article_text and categories:
    print(f"Categories: {categories}")
    
    # Create sentences from categories
    category_sentences = create_sentences_from_categories(categories)
    print("\nSentences from categories:")
    for sentence in category_sentences:
        print(f"- {sentence}")
    
    # Extract key sentences from the article
    key_sentences = extract_key_sentences(article_text)
    print("\nKey sentences from the article:")
    for i, sentence in enumerate(key_sentences, 1):
        print(f"{i}. {sentence}")
    
    # Show external links for more data sources
    if external_links:
        print("\nExternal links for more data:")
        for i, link in enumerate(external_links, 1):
            print(f"{i}. {link}")
    else:
        print("\nNo external links found in this article.")
    # Extract potential references from text
    references = get_references_from_text(article_text)
    if references:
        print("\nPotential references found in text:")
        for i, ref in enumerate(references, 1):
            print(f"{i}. {ref}")
    
    # Combine categories and article content
    combined_text = " ".join(categories) + " " + article_text[:500]  # First 500 chars of article
    blob = TextBlob(combined_text)
    words = blob.words
    print(f"\nExtracted words from combined content: {set(words[:20])}")  # Show first 20 unique words
    
else:
    print("Page not found using web scraping")

# Additional functionality: Get related Wikipedia pages
print("\n" + "="*50)
print("Additional data collection options:")
print("1. Get related Wikipedia pages")
print("2. Search for academic papers on this topic")
print("3. Find news articles about this topic")
print("4. Extract data from external links")

# You could expand this section to implement these options
# For example, using APIs like Google Scholar, News API, etc.