app.py

import streamlit as st
from scrapegraphai.graphs import SmartScraperGraph
import json

# Setup the Streamlit interface
st.title("Smart Scraper AI Interface")
prompt = st.text_input("Enter your query", value="List me all the articles")
source_url = st.text_input("Enter the source URL", value="https://perinim.github.io/projects")

if st.button("Fetch Data"):
    # Access API keys securely (ensure you've set this in Hugging Face Secrets)
    OPENAI_API_KEY = st.secrets["OPENAI_API_KEY"]
    
    # Define the configuration for the SmartScraperGraph
    graph_config = {
        "llm": {
            "api_key": OPENAI_API_KEY,
            "model": "gpt-3.5-turbo",
        },
    }

    # Create the SmartScraperGraph instance dynamically
    smart_scraper_graph = SmartScraperGraph(
        prompt=prompt,
        source=source_url,
        config=graph_config
    )

    try:
        # Run the graph to fetch results
        result = smart_scraper_graph.run()
        # Convert the result to a JSON string with indentation for better readability
        output = json.dumps(result, indent=2)
        # Display each line of the JSON output
        st.text_area("Result", value=output, height=300)
    except Exception as e:
        st.error(f"An error occurred: {e}")