backend/agents/normalizer.py

from datetime import datetime
import re

def clean_html(raw_html):
    """Remove HTML tags from a string."""
    if not raw_html or not isinstance(raw_html, str):
        return ""
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', raw_html)
    return cleantext.strip()

def normalize_job_data(job_data, source):
    """
    Unifies job data from different sources into a common schema.

    Args:
        job_data (dict): The raw job data from an API.
        source (str): The name of the job source (e.g., "RemoteOK", "LinkedIn").

    Returns:
        dict: A dictionary with the normalized job data.
    """
    if source == "RemoteOK":
        date_posted_str = 'N/A'
        try:
            # Safely convert timestamp
            timestamp = int(job_data.get('date', 0))
            if timestamp > 0:
                date_posted_str = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d')
        except (ValueError, TypeError, OSError):
            pass

        return {
            'id': job_data.get('id', ''),
            'title': job_data.get('position', 'N/A'),
            'company': job_data.get('company', 'N/A'),
            'location': job_data.get('location', 'Remote'),
            'description': clean_html(job_data.get('description', '')),
            'url': job_data.get('url', ''),
            'date_posted': date_posted_str,
            'source': source
        }

    elif source == "LinkedIn (Stub)":
        return {
            'id': f"linkedin_{job_data.get('title', '').replace(' ', '_')}",
            'title': job_data.get('title', 'N/A'),
            'company': job_data.get('company_name', 'N/A'),
            'location': job_data.get('location', 'Remote'),
            'description': clean_html(job_data.get('description', '')),
            'url': job_data.get('job_url', ''),
            'date_posted': job_data.get('posted_at', 'N/A'),
            'source': source
        }

    elif source == "Upwork (Stub)":
        return {
            'id': f"upwork_{job_data.get('title', '').replace(' ', '_')}",
            'title': job_data.get('title', 'N/A'),
            'company': 'Upwork Client',
            'location': job_data.get('client', {}).get('country', 'Remote'),
            'description': clean_html(job_data.get('snippet', '')),
            'url': job_data.get('url', ''),
            'date_posted': job_data.get('published_on', 'N/A').split('T')[0],
            'source': source
        }

    elif source == "Freelancer (Stub)":
        return {
            'id': f"freelancer_{job_data.get('title', '').replace(' ', '_')}",
            'title': job_data.get('title', 'N/A'),
            'company': 'Freelancer Client',
            'location': job_data.get('job_details', {}).get('country', 'Remote'),
            'description': clean_html(job_data.get('description', '')),
            'url': job_data.get('link', ''),
            'date_posted': datetime.fromtimestamp(job_data.get('submitdate', 0)).strftime('%Y-%m-%d'),
            'source': source
        }

    else:
        # Generic fallback
        return {
            'id': job_data.get('id', ''),
            'title': job_data.get('title', 'N/A'),
            'company': job_data.get('company', 'N/A'),
            'location': job_data.get('location', 'Remote'),
            'description': clean_html(job_data.get('description', '')),
            'url': job_data.get('url', ''),
            'date_posted': 'N/A',
            'source': source
        }