""" Usage: python run_scrape.py # scrape CA (default) python run_scrape.py TX 3 # scrape TX, 3 pages python run_scrape.py CA 5 proxy # override mode for this run Set SCRAPER_MODE in .env to switch modes permanently: rebrowser → free, no proxy, headless (default) proxy → set PROXY_URL in .env scraper_api → set SCRAPER_API_KEY in .env (free 1000 req/month) """ import asyncio import sys sys.path.insert(0, ".") from app.scrapers.zillow import ZillowScraper from app.db.session import AsyncSessionLocal, init_db from app.services import property_service from app.core.config import settings async def main(): state = sys.argv[1].upper() if len(sys.argv) > 1 else "CA" max_pages = int(sys.argv[2]) if len(sys.argv) > 2 else 2 # Optional: override mode from CLI without editing .env if len(sys.argv) > 3: settings.SCRAPER_MODE = sys.argv[3] print(f"State: {state}") print(f"Max pages: {max_pages}") print(f"Scraper mode: {settings.SCRAPER_MODE}") print(f"Proxy: {settings.PROXY_URL or '(none)'}") print(f"ScraperAPI: {'set' if settings.SCRAPER_API_KEY else '(none)'}") print("-" * 50) await init_db() async with ZillowScraper() as scraper: items = await scraper.scrape_state(state, max_pages=max_pages) print(f"\nScraped: {len(items)} properties") if items: async with AsyncSessionLocal() as db: saved = await property_service.bulk_upsert(db, items) print(f"Saved: {saved} records\n") print("Sample properties:") for p in items[:8]: price_str = f"${p.price:,.0f}" if p.price else "N/A" print(f" [{p.property_id}] {p.address} | {price_str} | {p.bedrooms}bd/{p.bathrooms}ba") else: print("\nNo results — IP may be temporarily blocked.") print("Wait 10-15 min and retry, OR switch mode:\n") print(" Option A (free): set SCRAPER_API_KEY in .env, then SCRAPER_MODE=scraper_api") print(" Sign up: https://www.scraperapi.com (1000 req/month free)") print(" Option B (cheap): set PROXY_URL in .env, then SCRAPER_MODE=proxy") print(" IPRoyal residential: https://iproyal.com (~$1.75/GB)") asyncio.run(main())