scrape.py

import streamlit as st
from scrapegraphai.graphs import SmartScraperGraph

st.set_page_config(page_title="Web Scraping AI Agent", layout="centered")

st.title("🌐 Web Scraping AI Agent")
st.caption("This app allows you to scrape a site using LLMs (Ollama / OpenAI).")

# Graph config
graph_config = {
    "llm": {
        "model": "ollama/llama3",
        "temperature": 0,
        "format": "json",
        "base_url": "http://localhost:11434",
    },
    "embeddings": {
        "model": "ollama/nomic-embed-text",
        "base_url": "http://localhost:11434",
    },
    "browser": "chromium",
    "playwright": {
        "headless": True,     # don’t open window
        "timeout": 20000      # stop after 20s
    },
    "verbose": True,
}


# Inputs
url = st.text_input("🔗 Enter URL you want to scrape:")
user_prompt = st.text_area("📝 What do you want the AI Agent to scrape from the site?")

if st.button("🚀 Scrape"):
    if not url or not user_prompt:
        st.warning("⚠️ Please enter both a URL and a prompt.")
    else:
        try:
            smart_scraper_graph = SmartScraperGraph(
                prompt=user_prompt,
                source=url,
                config=graph_config
            )
            with st.spinner("Scraping in progress... ⏳"):
                result = smart_scraper_graph.run()
            st.success("✅ Scraping complete!")
            st.json(result)  # Pretty JSON output
        except Exception as e:
            st.error(f"❌ Error: {str(e)}")