backend/scraper_template.py

"""
Real Estate Data Scraper Template
This is a production-ready blueprint for scraping PropertyFinder or Aqarmap.
"""
import requests
from bs4 import BeautifulSoup
import json
import time

def scrape_property_finder_egypt():
    print("[*] Starting Data Ingestion Pipeline...")
    
    # Target URL (Example for New Cairo Villas)
    url = "https://www.propertyfinder.eg/en/search?c=1&l=4&ob=mr&page=1"
    
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }

    try:
        response = requests.get(url, headers=headers)
        if response.status_code != 200:
            print(f"[!] Failed to fetch data: HTTP {response.status_code}")
            return
            
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # NOTE: The actual CSS classes change frequently. 
        # You will need to inspect the live site to get the exact classes.
        cards = soup.find_all('div', class_='property-card-module_property-card__row__t2p_f')
        
        properties = []
        for card in cards:
            title = card.find('h2').text.strip() if card.find('h2') else "Unknown"
            price = card.find('p', class_='price').text.strip() if card.find('p', class_='price') else "0"
            location = card.find('span', class_='location').text.strip() if card.find('span', class_='location') else "Unknown"
            
            # Extract Image URL
            img_tag = card.find('img')
            image_url = img_tag['src'] if img_tag and 'src' in img_tag.attrs else ""

            properties.append({
                "title": title,
                "price": price,
                "location": location,
                "image": image_url,
                "status": "للبيع",
                "source": "PropertyFinder"
            })
            
        # Save to database (or JSON for now)
        with open('raw_properties.json', 'w', encoding='utf-8') as f:
            json.dump(properties, f, ensure_ascii=False, indent=4)
            
        print(f"[+] Scraped {len(properties)} properties successfully.")
        
    except Exception as e:
        print(f"[!] Error during scraping: {e}")

if __name__ == "__main__":
    scrape_property_finder_egypt()