routers/website.py

"""
Website Router — Scrapes websites and summarizes them.

POST /api/parse-url
"""
import time
import requests
from bs4 import BeautifulSoup
from fastapi import APIRouter, HTTPException, Header
from pydantic import BaseModel

from models.schemas import (
    ExtractionResponse,
    ExtractionMetadata,
    ExtractedField,
    FileType,
    ProcessingLane,
    DocumentType,
)
from services.summarizer import generate_summary
from services.tier_manager import record_usage

router = APIRouter(prefix="/api", tags=["website"])

class ParseUrlRequest(BaseModel):
    url: str
    document_type: str = "free_text"

@router.post("/parse-url", response_model=ExtractionResponse)
async def parse_url(
    request: ParseUrlRequest,
    x_session_token: str = Header(default="anonymous"),
    x_user_registered: str = Header(default="false"),
):
    start_time = time.time()
    is_registered = x_user_registered.lower() == "true"
    doc_type_enum = DocumentType(request.document_type) if request.document_type in [e.value for e in DocumentType] else DocumentType.FREE_TEXT

    if not request.url.startswith("http"):
        request.url = "https://" + request.url

    # Try to fetch the URL
    try:
        # Add a common User-Agent to avoid simple blocks
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
        }
        response = requests.get(request.url, headers=headers, timeout=10)
        response.raise_for_status()
    except Exception as e:
        raise HTTPException(status_code=400, detail=f"Failed to fetch URL: {str(e)}")

    # Parse with BeautifulSoup
    soup = BeautifulSoup(response.content, 'html.parser')
    
    # Remove script and style elements
    for script in soup(["script", "style", "nav", "footer", "header"]):
        script.extract()

    # Get text
    raw_text = soup.get_text(separator='\n')
    
    # Break into lines and remove leading and trailing space on each
    lines = (line.strip() for line in raw_text.splitlines())
    # Break multi-headlines into a line each
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    # Drop blank lines
    raw_text = '\n'.join(chunk for chunk in chunks if chunk)

    fields = [ExtractedField(name="Website Content", value=raw_text, field_type="text", confidence=1.0)]

    # Generate AI Summary
    summary = generate_summary(raw_text, is_registered=is_registered)

    record_usage(x_session_token)

    processing_time = int((time.time() - start_time) * 1000)

    return ExtractionResponse(
        success=True,
        fields=fields,
        summary=summary,
        metadata=ExtractionMetadata(
            filename=request.url,
            file_type=FileType.URL,
            processing_lane=ProcessingLane.URL_PARSER,
            document_type=doc_type_enum,
            page_count=1,
            processing_time_ms=processing_time,
        ),
        message=f"Scraped and summarized {request.url} in {processing_time}ms",
    )