app.py

import gradio as gr
import asyncio
import os
import shutil
import pandas as pd
from all_model_scrapper import AllModelScrapper
from crawler.state import CrawlerState

# Global flag to stop running crawl
stop_crawl_flag = False

async def run_scraper_ui(url, max_pages, max_depth, concurrency, min_delay, max_delay):
    global stop_crawl_flag
    stop_crawl_flag = False
    
    db_path = "hf_crawler_state.db"
    
    # Remove existing db if fresh run
    if os.path.exists(db_path):
        try:
            os.remove(db_path)
        except Exception:
            pass

    scrapper = AllModelScrapper(
        base_url=url,
        db_path=db_path,
        max_depth=int(max_depth),
        concurrency_limit=int(concurrency),
        min_delay=float(min_delay),
        max_delay=float(max_delay),
        max_pages=int(max_pages) if max_pages else None
    )
    
    # We run the async loop and update Gradio live
    scrapper.config.respect_robots_txt = False
    
    from crawler.engine import CrawlEngine
    engine = CrawlEngine(scrapper.config, db_path=db_path)
    
    active_tasks = set()
    
    yield "Starting crawl...", None, None
    
    while not stop_crawl_flag:
        if scrapper.config.max_pages and engine.pages_scraped_count >= scrapper.config.max_pages:
            break
            
        free_slots = scrapper.config.concurrency_limit - len(active_tasks)
        for _ in range(free_slots):
            next_item = engine.state.get_next_url()
            if next_item:
                target_url, depth = next_item
                task = asyncio.create_task(engine._process_url(target_url, depth))
                active_tasks.add(task)
                task.add_done_callback(active_tasks.discard)
            else:
                break
                
        if not active_tasks:
            next_item = engine.state.get_next_url()
            if not next_item:
                break
                
        # Get stats
        stats = engine.state.get_stats()
        status_msg = f"Crawling: Scraped {stats['scraped']} pages | Pending: {stats['pending']} | Visited (Failed): {stats['failed']}"
        yield status_msg, None, None
        
        await asyncio.sleep(0.5)
        
    await engine.client.close()
    
    stats = engine.state.get_stats()
    scraped_data = engine.state.get_all_scraped_data()
    
    if scraped_data:
        df = pd.DataFrame(scraped_data)
        # Save to csv for download
        csv_path = "scraped_results.csv"
        df.to_csv(csv_path, index=False, encoding="utf-8-sig")
        
        # Format table preview (first 10 records)
        preview_df = df.head(10)[["url", "title", "h1_headers", "scraped_at"]]
        
        final_msg = f"Crawl finished. Scraped {stats['scraped']} pages successfully."
        if stop_crawl_flag:
            final_msg = f"Crawl stopped by user. Scraped {stats['scraped']} pages."
            
        yield final_msg, preview_df, csv_path
    else:
        yield "Crawl complete, but no data was extracted.", None, None

def stop_crawl():
    global stop_crawl_flag
    stop_crawl_flag = True
    return "Stopping crawler..."

with gr.Blocks(title="All Model Scrapper") as demo:
    gr.Markdown("# All Model Scrapper - Web Crawler Engine")
    gr.Markdown("Enter a URL and start scraping content. This model uses dynamic headers and anti-blocking techniques to bypass crawler detection.")
    
    with gr.Row():
        with gr.Column(scale=1):
            url_input = gr.Textbox(label="Target URL", placeholder="https://example.com", value="https://books.toscrape.com/")
            max_pages = gr.Number(label="Max Pages to Scrape (0 or empty for unlimited)", value=5, precision=0)
            max_depth = gr.Number(label="Max Crawl Depth", value=3, precision=0)
            concurrency = gr.Slider(label="Concurrency Limit", minimum=1, maximum=5, value=2, step=1)
            min_delay = gr.Slider(label="Minimum Request Delay (sec)", minimum=0.5, maximum=5.0, value=1.5, step=0.5)
            max_delay = gr.Slider(label="Maximum Request Delay (sec)", minimum=1.0, maximum=10.0, value=4.0, step=0.5)
            
            with gr.Row():
                start_btn = gr.Button("Start Scraping", variant="primary")
                stop_btn = gr.Button("Stop Scraping", variant="stop")
                
        with gr.Column(scale=2):
            status_output = gr.Textbox(label="Status / Console Logs", interactive=False)
            file_output = gr.File(label="Download Scraped CSV Data")
            table_preview = gr.Dataframe(label="Scraped Data Preview (First 10 rows)", interactive=False)
            
    start_btn.click(
        fn=run_scraper_ui,
        inputs=[url_input, max_pages, max_depth, concurrency, min_delay, max_delay],
        outputs=[status_output, table_preview, file_output]
    )
    
    stop_btn.click(
        fn=stop_crawl,
        inputs=[],
        outputs=[status_output]
    )

if __name__ == "__main__":
    demo.queue().launch()