alias_sync.py

import os
import tempfile
import pandas as pd
from huggingface_hub import HfApi, hf_hub_download

def sync_aliases(token: str, repo: str):
    """
    Downloads all master parquets and the current school_aliases.csv,
    syncs all multi-name schools, and re-uploads the CSV.
    """
    print("Starting automatic CSV alias sync...")
    api = HfApi(token=token)
    
    # 1. Get all files
    files = api.list_repo_files(repo_id=repo, repo_type="dataset")
    
    # 2. Download Old Master
    print("Downloading Old Master...")
    old_master_df = pd.DataFrame()
    try:
        if "data/baseline_master.parquet" in files:
            path = hf_hub_download(repo_id=repo, filename="data/baseline_master.parquet", repo_type="dataset", token=token, force_download=True)
            old_master_df = pd.read_parquet(path)
    except Exception as e:
        print(f"Error loading old master: {e}")

    # 3. Download all mapped masters
    mapped_files = sorted([f for f in files if f.startswith("scraped_data/mapped/mapped_master_") and f.endswith(".parquet")], reverse=True)
    
    mapped_dfs = {}
    for f in mapped_files:
        stem = f.split("/")[-1].replace(".parquet", "")
        print(f"Downloading {stem}...")
        try:
            path = hf_hub_download(repo_id=repo, filename=f, repo_type="dataset", token=token, force_download=True)
            mapped_dfs[stem] = pd.read_parquet(path)
        except Exception as e:
            print(f"Error loading {f}: {e}")

    # Helper for pretty label
    def _pretty_label(stem: str, index: int) -> tuple[str, str]:
        parts = stem.split("_")
        if len(parts) >= 6:
            year, month = parts[2], parts[3]
            ym = f"{year}-{month.capitalize()}"
            if index == 0:
                return f"Latest Scraped Master ({month.capitalize()}-{year})", ym
            return f"Past Scraped Master ({month.capitalize()}-{year})", ym
        return "Scraped Master", ""

    # Build UDISE dict
    udise_aliases = {}
    
    # Process old master
    if not old_master_df.empty:
        for _, row in old_master_df.iterrows():
            udise = str(row.get("School_Udise_Code__c", "")).strip()
            name = str(row.get("School_Name__c", "")).strip()
            if udise and name and udise != "nan" and name != "nan":
                if udise not in udise_aliases:
                    udise_aliases[udise] = {}
                udise_aliases[udise][name.upper()] = {
                    "name": name,
                    "source": "Old Master (baseline_master.parquet)",
                    "year_month": "2025"
                }
                
    # Process mapped masters
    for idx, (stem, df) in enumerate(mapped_dfs.items()):
        label, ym = _pretty_label(stem, idx)
        for _, row in df.iterrows():
            udise = str(row.get("School_Udise_Code__c", "")).strip()
            name = str(row.get("School_Name__c", "")).strip()
            if udise and name and udise != "nan" and name != "nan":
                if udise not in udise_aliases:
                    udise_aliases[udise] = {}
                if name.upper() not in udise_aliases[udise]:
                    udise_aliases[udise][name.upper()] = {
                        "name": name,
                        "source": label,
                        "year_month": ym
                    }
                    
    # Load existing CSV to preserve manual edits
    print("Loading existing CSV aliases...")
    try:
        path = hf_hub_download(repo_id=repo, filename="school_aliases.csv", repo_type="dataset", token=token, force_download=True)
        existing_csv = pd.read_csv(path)
        for _, row in existing_csv.iterrows():
            udise = str(row.get("UDISE_Code", "")).strip()
            name = str(row.get("Alias_Name", "")).strip()
            if udise and name and udise != "nan" and name != "nan":
                if udise not in udise_aliases:
                    udise_aliases[udise] = {}
                udise_aliases[udise][name.upper()] = {
                    "name": name,
                    "source": str(row.get("Source", "")),
                    "year_month": str(row.get("Year_Month", "")) if pd.notna(row.get("Year_Month")) else "",
                    "last_updated": str(row.get("Last_Updated", "")) if pd.notna(row.get("Last_Updated")) else ""
                }
    except Exception as e:
        print(f"Could not load existing CSV: {e}")

    # Extract multi-name schools
    rows = []
    for udise, names_dict in udise_aliases.items():
        if len(names_dict) > 1:
            for upper_name, info in names_dict.items():
                rows.append({
                    "UDISE_Code": udise,
                    "Alias_Name": info["name"],
                    "Source": info["source"],
                    "Year_Month": info["year_month"],
                    "Last_Updated": info.get("last_updated", "")
                })

    if not rows:
        print("No multi-name schools found.")
        return

    df = pd.DataFrame(rows, columns=["UDISE_Code", "Alias_Name", "Source", "Year_Month", "Last_Updated"])
    
    # Force string type to prevent sorting errors
    df["UDISE_Code"] = df["UDISE_Code"].astype(str)
    
    # Sort by UDISE code so that all identical schools are grouped together perfectly
    df = df.sort_values(by=["UDISE_Code", "Source"]).reset_index(drop=True)

    print(f"Uploading {len(df)} synced aliases to HuggingFace...")
    try:
        with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False, encoding="utf-8") as f:
            df.to_csv(f.name, index=False)
            tmp_path = f.name
            
        api.upload_file(
            path_or_fileobj=tmp_path,
            path_in_repo="school_aliases.csv",
            repo_id=repo,
            repo_type="dataset",
            token=token,
            commit_message=f"Auto-sync aliases after building mapped master ({len(df)} aliases)"
        )
        if os.path.exists(tmp_path):
            os.remove(tmp_path)
        print("CSV Sync complete!")
    except Exception as e:
        print(f"Failed to upload CSV: {e}")