run_scrape.py

"""
Usage:
  python run_scrape.py              # scrape CA (default)
  python run_scrape.py TX 3         # scrape TX, 3 pages
  python run_scrape.py CA 5 proxy   # override mode for this run

Set SCRAPER_MODE in .env to switch modes permanently:
  rebrowser   → free, no proxy, headless (default)
  proxy       → set PROXY_URL in .env
  scraper_api → set SCRAPER_API_KEY in .env (free 1000 req/month)
"""
import asyncio
import sys
sys.path.insert(0, ".")

from app.scrapers.zillow import ZillowScraper
from app.db.session import AsyncSessionLocal, init_db
from app.services import property_service
from app.core.config import settings


async def main():
    state     = sys.argv[1].upper() if len(sys.argv) > 1 else "CA"
    max_pages = int(sys.argv[2])    if len(sys.argv) > 2 else 2
    # Optional: override mode from CLI without editing .env
    if len(sys.argv) > 3:
        settings.SCRAPER_MODE = sys.argv[3]

    print(f"State:        {state}")
    print(f"Max pages:    {max_pages}")
    print(f"Scraper mode: {settings.SCRAPER_MODE}")
    print(f"Proxy:        {settings.PROXY_URL or '(none)'}")
    print(f"ScraperAPI:   {'set' if settings.SCRAPER_API_KEY else '(none)'}")
    print("-" * 50)

    await init_db()

    async with ZillowScraper() as scraper:
        items = await scraper.scrape_state(state, max_pages=max_pages)

    print(f"\nScraped: {len(items)} properties")

    if items:
        async with AsyncSessionLocal() as db:
            saved = await property_service.bulk_upsert(db, items)
        print(f"Saved:   {saved} records\n")

        print("Sample properties:")
        for p in items[:8]:
            price_str = f"${p.price:,.0f}" if p.price else "N/A"
            print(f"  [{p.property_id}] {p.address} | {price_str} | {p.bedrooms}bd/{p.bathrooms}ba")
    else:
        print("\nNo results — IP may be temporarily blocked.")
        print("Wait 10-15 min and retry, OR switch mode:\n")
        print("  Option A (free):  set SCRAPER_API_KEY in .env, then SCRAPER_MODE=scraper_api")
        print("                    Sign up: https://www.scraperapi.com  (1000 req/month free)")
        print("  Option B (cheap): set PROXY_URL in .env, then SCRAPER_MODE=proxy")
        print("                    IPRoyal residential: https://iproyal.com (~$1.75/GB)")


asyncio.run(main())