scripts/repair_localized_data.py

import pandas as pd
import os
import glob
import sys

# Force UTF-8 output
try:
    if hasattr(sys.stdout, 'reconfigure'):
        sys.stdout.reconfigure(encoding='utf-8')
except:
    pass

def repair():
    data_dir = "backend/data"
    master_path = os.path.join(data_dir, "seoul_api_cache.parquet")
    
    if not os.path.exists(master_path):
        print("MASTER_NOT_FOUND")
        return

    df_ko = pd.read_parquet(master_path, engine='pyarrow')
    
    # Map: link -> {type, region, is_free}
    # Columns in master are Korean
    ref_df = df_ko[['link', 'type', 'region', 'is_free']].copy()
    ref_df.columns = ['link', 'type_ko_ref', 'region_ko_ref', 'is_free_ko_ref']
    ref_df = ref_df.drop_duplicates(subset=['link'])

    localized_files = glob.glob(os.path.join(data_dir, "seoul_api_cache_*.parquet"))
    
    results = []
    for path in localized_files:
        lang = os.path.basename(path).replace("seoul_api_cache_", "").replace(".parquet", "")
        if lang == "ko": continue
        
        try:
            df_lang = pd.read_parquet(path, engine='pyarrow')
            
            # CRITICAL FIX: Drop existing reference columns to avoid merge collisions
            cols_to_drop = [c for c in ['type_ko', 'region_ko', 'is_free_ko', 'type_ko_ref', 'region_ko_ref', 'is_free_ko_ref'] if c in df_lang.columns]
            if cols_to_drop:
                df_lang = df_lang.drop(columns=cols_to_drop)
            
            # Perform clean Merge
            repaired_df = pd.merge(df_lang, ref_df, on='link', how='left')
            
            # Rename reference columns to final filter keys
            repaired_df = repaired_df.rename(columns={
                'type_ko_ref': 'type_ko',
                'region_ko_ref': 'region_ko',
                'is_free_ko_ref': 'is_free_ko'
            })
            
            # Fill NaNs with empty string or category default to prevent isin failures
            repaired_df['type_ko'] = repaired_df['type_ko'].fillna("기타")
            repaired_df['region_ko'] = repaired_df['region_ko'].fillna("종로구")
            repaired_df['is_free_ko'] = repaired_df['is_free_ko'].fillna("무료")
            
            repaired_df.to_parquet(path, engine='pyarrow', compression='snappy', index=False)
            results.append(f"{lang}:{len(repaired_df)}")
        except Exception as e:
            results.append(f"{lang}:FAILED")

    print("|".join(results))

if __name__ == "__main__":
    repair()