src/option1.py

import os
import requests
from bs4 import BeautifulSoup
from dotenv import load_dotenv

try:
    from src.app_hf import get_llm
except ImportError:
    from app_hf import get_llm

# Setup LLM (Gemini first, then Hugging Face fallback)
load_dotenv()
hf_token = os.getenv('HF_TOKEN') or os.getenv('HUGGINGFACEHUB_API_TOKEN')
os.environ.setdefault('HUGGINGFACEHUB_API_TOKEN', hf_token)
llm = get_llm(model_name="Qwen/Qwen2.5-7B-Instruct", task="text-generation")

def get_first_result_url(query: str) -> str:
    """Gets the URL of the top result from DuckDuckGo HTML version."""
    url = "https://html.duckduckgo.com/html/"
    headers = {"User-Agent": "Mozilla/5.0"}
    data = {"q": query}
    
    response = requests.post(url, headers=headers, data=data)
    soup = BeautifulSoup(response.text, "html.parser")
    
    # Grab the first link
    first_result = soup.find('a', class_='result__a')
    if first_result:
        return first_result['href']
    return None

def crawl_website(url: str) -> str:
    """Scrapes the main text content from a given URL."""
    print(f"Crawling: {url} ...")
    try:
        response = requests.get(url, timeout=10, headers={"User-Agent": "Mozilla/5.0"})
        soup = BeautifulSoup(response.text, "html.parser")
        
        # Remove junk
        for tag in soup(["nav", "footer", "script", "style", "aside", "header"]):
            tag.decompose()
            
        return soup.get_text(separator=' ', strip=True)[:5000]
    except Exception as e:
        return f"Error crawling {url}: {e}"

def search_and_crawl(query: str):
    # 1. Search
    target_url = get_first_result_url(query)
    if not target_url:
        print("No results found.")
        return

    # 2. Crawl
    content = crawl_website(target_url)
    
    # 3. Generate Answer
    prompt = f"Answer the user query based on the following text.\n\nQuery: {query}\n\nText: {content[:3000]}"
    print("\nGenerating answer...")
    response = llm.invoke(prompt)
    answer = getattr(response, "content", response)
    print(answer)

# Run it
search_and_crawl("what is deepagent? with sample text")