scripts/scraper.py

#!/usr/bin/env python3
"""
Scrape a website and convert to Markdown for the AIbrahim knowledge base.

Usage:
    python scripts/scraper.py

Requires:
    pip install firecrawl-py
    FIRECRAWL_API_KEY environment variable
"""

import os
from pathlib import Path
from dotenv import load_dotenv

# Try to import firecrawl, provide helpful error if not installed
try:
    from firecrawl import FirecrawlApp
except ImportError:
    print("❌ firecrawl-py not installed. Run: pip install firecrawl-py")
    exit(1)

load_dotenv()

# Configuration
PROJECT_ROOT = Path(__file__).parent.parent
OUTPUT_FILE = PROJECT_ROOT / "knowledge_base_md" / "website.md"


def main():
    api_key = os.getenv("FIRECRAWL_API_KEY")
    
    if not api_key:
        print("❌ FIRECRAWL_API_KEY not set in environment")
        return
    
    # Initialize
    app = FirecrawlApp(api_key=api_key)
    
    # Crawl the site
    print("🕷️ Crawling website... (This may take a few minutes)")
    crawl_result = app.crawl(
        'https://morawa-dev.github.io/albarka.github.io/', 
        limit=100,  # Safety limit
        scrape_options={
            'formats': ['markdown']
        }
    )
    
    # Save the data
    if crawl_result and crawl_result.data:
        full_knowledge_base = ""
        
        for page in crawl_result.data:
            url = page.metadata.source_url if page.metadata else 'Unknown URL'
            full_knowledge_base += f"\n\n# SOURCE: {url}\n"
            full_knowledge_base += page.markdown or ''
        
        OUTPUT_FILE.parent.mkdir(exist_ok=True)
        OUTPUT_FILE.write_text(full_knowledge_base, encoding="utf-8")
            
        print(f"✅ Success! Website saved to '{OUTPUT_FILE}'")
    else:
        print("❌ Crawl Failed:", crawl_result)


if __name__ == "__main__":
    main()