SmartScraper / app.py
sergioska's picture
add scrape graph script
5465fb7
import streamlit as st
from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM
from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.utils import prettify_exec_info
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1")
modelNomic = AutoModel.from_pretrained("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True)
graph_config = {
"llm": {
"model-instance": model,
"temperature": 1,
"format": "json", # Ollama needs the format to be specified explicitly
"model_tokens": 4096, # depending on the model set context length
},
"embeddings": {
"model-instance": modelNomic,
"temperature": 0,
}
}
# ************************************************
# Create the SmartScraperGraph instance and run it
# ************************************************
smart_scraper_graph = SmartScraperGraph(
prompt="List me shoes in first page with names, prices and image urls",
# also accepts a string with the already downloaded HTML code
source="https://www.footlocker.co.uk/en/category/sale/men.html",
config=graph_config
)
result = smart_scraper_graph.run()
print(result)
x = st.slider('Select a value')
st.write(x, 'squared is', x * x)