File size: 1,363 Bytes
c914208 35e215c 32528da 35e215c 32528da 35e215c 32528da | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | import streamlit as st
import subprocess
import json
import os
import uuid
st.title("Advanced Scrapy Scraper")
url = st.text_input("Start URL", "https://example.com")
run = st.button("Run Scrape")
if run and url:
output_file = f"output_{uuid.uuid4().hex}.json" # unique file per run
with st.spinner("Scraping..."):
cmd = [
"scrapy", "crawl", "advanced",
"-a", f"start_url={url}",
"-o", output_file,
"--nolog"
]
result = subprocess.run(
cmd,
cwd=".",
capture_output=True,
text=True
)
# Check if Scrapy failed
if result.returncode != 0:
st.error("Scrapy failed")
st.text(result.stderr)
else:
if os.path.exists(output_file):
try:
with open(output_file, "r") as f:
data = json.load(f)
st.success("Scrape complete")
st.json(data)
st.download_button(
"Download JSON",
json.dumps(data, indent=2),
"results.json",
"application/json"
)
except json.JSONDecodeError:
st.error("Output file is not valid JSON")
else:
st.error("No output file generated") |