expand_languages.py

import os
import json

iedid_dir = 'c:/Users/toecm/Documents/Pure Versation/pureversation/backend/IEDID'
lab_profiles_dir = 'c:/Users/toecm/Documents/Pure Versation/pureversation/backend/lab_profiles'

# Top languages and representative dialects
top_languages = {
    'Mandarin Chinese': ['Standard Mandarin', 'Beijing Dialect', 'Taiwanese Mandarin', 'Singaporean Mandarin'],
    'Spanish': ['Castilian Spanish', 'Mexican Spanish', 'Argentine Spanish', 'Colombian Spanish', 'Caribbean Spanish'],
    'Hindi': ['Standard Hindi', 'Bhojpuri', 'Awadhi', 'Haryanvi', 'Rajasthani'],
    'Arabic': ['Modern Standard Arabic', 'Egyptian Arabic', 'Levantine Arabic', 'Gulf Arabic', 'Maghrebi Arabic'],
    'Bengali': ['Rarhi', 'Varendri', 'Kamarupi', 'Bangali', 'Sylheti'],
    'Portuguese': ['Brazilian Portuguese', 'European Portuguese', 'Angolan Portuguese', 'Mozambican Portuguese'],
    'Russian': ['Northern Russian', 'Central Russian', 'Southern Russian'],
    'Japanese': ['Kanto', 'Kansai', 'Kyushu', 'Tohoku'],
    'Western Punjabi': ['Majhi', 'Pothohari', 'Jhangochi', 'Shahpuri'],
    'Marathi': ['Standard Marathi', 'Varhadi', 'Konkani', 'Deshi'],
    'Telugu': ['Coastal Telugu', 'Rayalaseema', 'Telangana'],
    'Wu Chinese': ['Shanghainese', 'Suzhounese', 'Wenzhounese'],
    'Turkish': ['Istanbul Turkish', 'Rumelian', 'Anatolian'],
    'Korean': ['Seoul Dialect', 'Gyeongsang Satoori', 'Jeolla Satoori', 'Chungcheong Satoori', 'Jeju Dialect'],
    'French': ['Metropolitan French', 'Quebec French', 'African French', 'Belgian French', 'Swiss French'],
    'German': ['High German', 'Low German', 'Bavarian', 'Austrian German', 'Swiss German'],
    'Vietnamese': ['Northern Vietnamese', 'Central Vietnamese', 'Southern Vietnamese'],
    'Tamil': ['Central Tamil', 'Madurai Tamil', 'Kongu Tamil', 'Chennai Tamil', 'Sri Lankan Tamil'],
    'Yue Chinese (Cantonese)': ['Guangzhou Cantonese', 'Hong Kong Cantonese', 'Macau Cantonese'],
    'Urdu': ['Dakhini', 'Rekhta'],
    'Javanese': ['Central Javanese', 'Eastern Javanese', 'Western Javanese'],
    'Italian': ['Standard Italian', 'Neapolitan', 'Sicilian', 'Venetian'],
    'Egyptian Arabic': ['Cairene', 'Saidi', 'Alexandrian'],
    'Gujarati': ['Standard Gujarati', 'Kathiawadi', 'Surati', 'Pattani'],
    'Iranian Persian': ['Tehrani', 'Shirazi', 'Isfahani', 'Mashhadi'],
    'Bhojpuri': ['Northern Bhojpuri', 'Southern Bhojpuri', 'Western Bhojpuri'],
    'Min Nan Chinese': ['Hokkien', 'Teochew', 'Hainanese'],
    'Hakka Chinese': ['Meixian', 'Hailu', 'Sixian'],
    'Jin Chinese': ['Bingzhou', 'Lüliang', 'Zhangjiakou-Hohhot'],
    'Hausa': ['Kano', 'Sokoto', 'Zaria', 'Katsina'],
    'Kannada': ['Mysore', 'Hubli', 'Mangalore', 'Kundapura'],
    'Indonesian': ['Standard Indonesian', 'Jakarta Indonesian', 'Papuan Malay'],
    'Polish': ['Greater Polish', 'Lesser Polish', 'Masovian', 'Silesian'],
    'Yoruba': ['Standard Yoruba', 'Ibadan', 'Egba', 'Ijebu'],
    'Xiang Chinese': ['Changyi', 'Loushao', 'Chenxü'],
    'Malayalam': ['South Kerala', 'Central Kerala', 'North Kerala'],
    'Odia': ['Mughalbandi', 'Sambalpuri', 'Ganjami', 'Desia'],
    'Maithili': ['Standard Maithili', 'Bajjika', 'Angika'],
    'Burmese': ['Standard Burmese', 'Mandalay', 'Yangon'],
    'Sunda': ['Priangan', 'Banten', 'Cirebon'],
    'Sudanese Arabic': ['Khartoum', 'Western Sudanese'],
    'Algerian Arabic': ['Algiers', 'Oran', 'Constantine'],
    'Moroccan Arabic': ['Rabat', 'Casablanca', 'Fes'],
    'Ukrainian': ['Northern Ukrainian', 'Southwestern Ukrainian', 'Southeastern Ukrainian'],
    'Igbo': ['Standard Igbo', 'Owerri', 'Onitsha', 'Umuahia', 'Orlu'],
    'Northern Uzbek': ['Tashkent', 'Fergana', 'Khorezm'],
    'Sindhi': ['Vicholi', 'Lari', 'Thari', 'Kachhi'],
    'North Levantine Arabic': ['Syrian', 'Lebanese'],
    'Romanian': ['Moldavian', 'Wallachian', 'Transylvanian'],
    'Tagalog': ['Manila', 'Batangas', 'Bulacan', 'Tayabas'],
    'Dutch': ['Hollandic', 'Brabantian', 'Limburgish'],
    'Saʽidi Arabic': ['Upper Egyptian'],
    'Gan Chinese': ['Nanchang', 'Yichun', 'Ji\'an'],
    'Amharic': ['Gondar', 'Gojjam', 'Shoa'],
    'Northern Pashto': ['Peshawari', 'Kabul'],
    'Magahi': ['Standard Magahi', 'Khortha'],
    'Thai': ['Central Thai', 'Khorat', 'Southern Thai'],
    'Saraiki': ['Multani', 'Derawali', 'Thali'],
    'Khmer': ['Standard Khmer', 'Northern Khmer', 'Western Khmer'],
    'Chhattisgarhi': ['Kedri', 'Khandi', 'Laria'],
    'Somali': ['Northern Somali', 'Benaadir', 'Maay'],
    'Malay': ['Standard Malay', 'Kelantanese', 'Terengganu'],
    'Cebuano': ['Standard Cebuano', 'Boholano', 'Leyteño'],
    'Nepali': ['Standard Nepali', 'Eastern Nepali', 'Western Nepali'],
    'Mesopotamian Arabic': ['Baghdadi', 'Basrawi', 'Moslawi'],
    'Assamese': ['Eastern Assamese', 'Central Assamese', 'Kamrupi', 'Goalpariya'],
    'Sinhalese': ['Standard Sinhala', 'Kandyan', 'Southern Sinhala'],
    'Northern Kurdish': ['Kurmanji', 'Badini'],
    'Hejazi Arabic': ['Meccan', 'Jeddawi', 'Medani'],
    'Nigerian Fulfulde': ['Kano', 'Katsina', 'Sokoto'],
    'Bavarian': ['North Bavarian', 'Central Bavarian', 'South Bavarian'],
    'South Azerbaijani': ['Tabrizi', 'Urmia'],
    'Greek': ['Standard Greek', 'Cypriot Greek', 'Cretan Greek', 'Pontic Greek'],
    'Chittagonian': ['Standard Chittagonian', 'Rohingya'],
    'Kazakh': ['Northeastern Kazakh', 'Southern Kazakh', 'Western Kazakh'],
    'Deccan': ['Standard Deccani'],
    'Hungarian': ['Alföld', 'Transdanubian', 'Székely'],
    'Kinyarwanda': ['Standard Kinyarwanda'],
    'Zulu': ['Standard Zulu', 'Lala', 'Qwabe'],
    'South Levantine Arabic': ['Palestinian', 'Jordanian'],
    'Tunisian Arabic': ['Tunis', 'Sfax', 'Sahel'],
    'Sanaani Arabic': ['Sana\'a'],
    'Min Bei Chinese': ['Jian\'ou'],
    'Southern Pashto': ['Kandahari', 'Quetta'],
    'Rundi': ['Standard Kirundi'],
    'Czech': ['Common Czech', 'Moravian', 'Silesian'],
    'Taʽizzi-Adeni Arabic': ['Ta\'izzi', 'Adeni'],
    'Uyghur': ['Central Uyghur', 'Hotan', 'Lop'],
    'Min Dong Chinese': ['Fuzhou'],
    'Sylheti': ['Standard Sylheti'],
    'Swahili': ['Kiunguja', 'Kimvita', 'Amu', 'Mrimima', 'Pemba']
}

for lang, dialects in top_languages.items():
    # 1. Create IEDID
    lang_data_dir = os.path.join(iedid_dir, lang, 'data')
    os.makedirs(lang_data_dir, exist_ok=True)
    for dialect in dialects:
        csv_path = os.path.join(lang_data_dir, f"{dialect}.csv")
        if not os.path.exists(csv_path):
            with open(csv_path, 'w', encoding='utf-8') as f:
                f.write("phrase,clarification\n")
    
    # 2. Create lab_profiles
    lang_persona_dir = os.path.join(lab_profiles_dir, lang)
    os.makedirs(lang_persona_dir, exist_ok=True)
    for dialect in dialects:
        persona_path = os.path.join(lang_persona_dir, f"{dialect} Persona.json")
        if not os.path.exists(persona_path):
            with open(persona_path, 'w', encoding='utf-8') as f:
                json.dump({"language": lang, "dialect": dialect, "system_prompt": f"You are a native speaker of {dialect} ({lang}, f)."}, f)
                # Just a minimal stub json
                f.write('{"name": "' + dialect + ' Persona"}')

print("Languages expanded successfully!")