"""Add English paraphrase + French aliases to the curated phrasebooks. Only touches the INPUT side (sources / source_fr / sources_fr). Bambara and Pular `target` strings are never modified. Idempotent: re-running merges without duplicating. Aborts if any alias would map to two different targets in the same phrasebook. python scripts/add_phrasebook_aliases.py # apply python scripts/add_phrasebook_aliases.py --check # dry run """ from __future__ import annotations import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from src.llm.phrasebook import _normalize # noqa: E402 DIR = Path(__file__).resolve().parent.parent / "configs" / "dialect_anchors" # canonical English source -> (english aliases, french canonical, french aliases) A: dict[str, tuple[list[str], str, list[str]]] = { "Hello / Thank you": (["hello", "hi", "thank you", "thanks"], "bonjour", ["salut", "merci"]), "Good morning": (["morning", "good morning to you"], "bonjour (le matin)", ["bon matin"]), "Good afternoon": (["afternoon"], "bon après-midi", []), "Good evening": (["evening"], "bonsoir", []), "Welcome": (["you are welcome here", "welcome home"], "bienvenue", ["soyez le bienvenu"]), "How are you?": (["how are you doing", "how is it going", "are you well"], "comment ça va ?", ["ça va ?", "comment allez-vous ?", "comment vas-tu ?"]), "Fine, no trouble": (["i am fine", "i'm fine", "fine"], "ça va bien", ["je vais bien"]), "How was the night?": (["did you sleep well", "how did you sleep"], "as-tu bien dormi ?", ["avez-vous bien dormi ?"]), "How was the work?": (["how was work", "how is the work"], "comment était le travail ?", []), "Well done": (["good job", "great work"], "bravo", ["bon travail"]), "What is your name?": (["what's your name", "your name"], "comment tu t'appelles ?", ["quel est ton nom ?", "comment vous appelez-vous ?"]), "My name is...": (["my name is", "i am called"], "je m'appelle", ["mon nom est"]), "Where are you from?": (["where do you come from"], "d'où viens-tu ?", ["d'où venez-vous ?"]), "I am from...": (["i am from", "i come from"], "je viens de", []), "What is your work?": (["what do you do", "what is your job"], "quel est ton travail ?", ["que fais-tu ?"]), "How is the family?": (["how is your family", "how's the family"], "comment va la famille ?", []), "How is your wife?": (["how's your wife"], "comment va ta femme ?", ["comment va votre femme ?"]), "How is your husband?": (["how's your husband"], "comment va ton mari ?", ["comment va votre mari ?"]), "How are the children?": (["how are the kids", "how are your children"], "comment vont les enfants ?", []), "How is the baby?": (["how's the baby"], "comment va le bébé ?", []), "They are fine": (["they are well", "they're fine"], "ils vont bien", []), "My father is well": (["my dad is fine"], "mon père va bien", []), "My mother is well": (["my mom is fine", "my mum is fine"], "ma mère va bien", []), "Are you married?": (["are you married"], "es-tu marié ?", ["êtes-vous marié ?"]), "I am hungry": (["i'm hungry"], "j'ai faim", []), "I am thirsty": (["i'm thirsty"], "j'ai soif", []), "I want to eat": (["i would like to eat"], "je veux manger", []), "Give me water": (["water please", "can i have water"], "donne-moi de l'eau", ["de l'eau s'il vous plaît"]), "I am full": (["i'm full", "i've eaten enough"], "je suis rassasié", []), "Bread": ([], "pain", []), "Rice": ([], "riz", []), "Meat": ([], "viande", []), "Tea": ([], "thé", []), "Sugar": ([], "sucre", []), "How is the farming?": (["how is the farm", "how are the crops"], "comment va le champ ?", ["comment va la culture ?"]), "It rained today": (["it rained", "there was rain today"], "il a plu aujourd'hui", []), "The field": (["field", "farm field"], "le champ", []), "Maize / Corn": (["maize", "corn"], "maïs", []), "Cow": (["a cow"], "vache", []), "Sheep": ([], "mouton", []), "Goat": ([], "chèvre", []), "Chicken": (["hen"], "poulet", ["poule"]), "Where is the hoe?": (["where's the hoe"], "où est la houe ?", ["où est la daba ?"]), "I am sick": (["i'm sick", "i am ill"], "je suis malade", []), "My head hurts": (["i have a headache"], "j'ai mal à la tête", []), "My stomach hurts": (["i have a stomach ache"], "j'ai mal au ventre", []), "I have fever": (["i have a fever"], "j'ai de la fièvre", []), "Where is the hospital?": (["where's the hospital"], "où est l'hôpital ?", []), "Where is the doctor?": (["where's the doctor"], "où est le médecin ?", ["où est le docteur ?"]), "Take the medicine": (["take your medicine"], "prends le médicament", []), "Drink this": ([], "bois ceci", ["bois ça"]), "Lie down": (["lay down"], "couche-toi", []), "Do you feel better?": (["are you feeling better"], "tu te sens mieux ?", ["ça va mieux ?"]), "How much?": (["how much is it", "what is the price"], "combien ?", ["c'est combien ?"]), "It is too much": (["it's too much", "too expensive"], "c'est trop cher", []), "Reduce it": (["lower the price"], "diminue le prix", ["baisse le prix"]), "I have no money": (["i don't have money"], "je n'ai pas d'argent", []), "Here is the money": (["here's the money"], "voici l'argent", []), "Market": (["the market"], "marché", ["le marché"]), "Shop": (["store"], "boutique", []), "Soap": ([], "savon", []), "Oil": (["cooking oil"], "huile", []), "Salt": ([], "sel", []), "Where is the road?": (["which way is the road"], "où est la route ?", []), "Is it far?": (["is it far away"], "c'est loin ?", []), "It is close": (["it's close", "it's nearby"], "c'est proche", []), "Turn right": ([], "tourne à droite", []), "Turn left": ([], "tourne à gauche", []), "Stop here": ([], "arrête-toi ici", []), "Let's go": (["let us go"], "on y va", ["allons-y"]), "Car": ([], "voiture", []), "Bus": ([], "bus", []), "Motorbike": (["motorcycle"], "moto", []), "I understand": (["i get it"], "je comprends", []), "I don't understand": (["i do not understand"], "je ne comprends pas", []), "Repeat it": (["please repeat", "say it again"], "répète", ["répétez s'il vous plaît"]), "Speak slowly": (["speak slower", "slow down please"], "parle lentement", ["parlez lentement"]), "Do you speak Bambara?": (["can you speak bambara"], "tu parles bambara ?", []), "A little": (["a bit"], "un peu", []), "I don't know": (["i do not know"], "je ne sais pas", []), "Yes": ([], "oui", []), "No": ([], "non", []), "Wait": (["wait a moment"], "attends", ["attendez"]), "Today": ([], "aujourd'hui", []), "Tomorrow": ([], "demain", []), "Yesterday": ([], "hier", []), "Now": (["right now"], "maintenant", []), "Later": ([], "plus tard", []), "Goodbye": (["bye"], "au revoir", []), "Until later": (["see you later"], "à plus tard", []), "Until tomorrow": (["see you tomorrow"], "à demain", []), "Have a good day": ([], "bonne journée", []), "Have a good night": (["good night"], "bonne nuit", []), "Go in peace": ([], "va en paix", []), "God bless you": ([], "que dieu te bénisse", []), "God willing": (["inshallah"], "si dieu le veut", []), "Thank God": (["praise god", "alhamdulillah"], "dieu merci", []), "Peace only": (["only peace"], "la paix seulement", []), # Pular-only keys "Peace only (Response)": (["peace only response"], "la paix seulement (réponse)", []), "Is there any trouble?": (["any trouble", "any problem"], "il y a un problème ?", []), "The food is sweet": (["the food is good", "the food is delicious"], "la nourriture est bonne", ["c'est bon"]), "We are working": (["we're working"], "nous travaillons", ["on travaille"]), "We are working (speaking to the person I'm working with)": ([], "", []), "We are working (speaking to another person not working with us)": ([], "", []), "No trouble": (["no problem"], "pas de problème", []), "How is the heat/weather?": (["how is the heat", "how is the weather"], "comment est la chaleur ?", ["quel temps fait-il ?"]), "Where are you coming from?": (["where are you coming from"], "d'où arrives-tu ?", []), "I am coming from...": (["i am coming from"], "j'arrive de", []), "I am from": (["i come from"], "je viens de", []), "I am a farmer": (["i'm a farmer"], "je suis cultivateur", ["je suis agriculteur"]), "How is the woman?": ([], "comment va la femme ?", []), "How is the man?": ([], "comment va l'homme ?", []), "Everyone is fine": (["everybody is fine", "all are well"], "tout le monde va bien", []), "How many children?": (["how many kids"], "combien d'enfants ?", []), "The food is good": (["the food is delicious"], "la nourriture est bonne", ["c'est bon"]), "Milk": ([], "lait", []), "Sour Cream": (["sour milk"], "lait caillé", []), "Hot water": ([], "eau chaude", []), "Cold water": ([], "eau froide", []), "Coffee": ([], "café", []), "The rain is good": (["good rain"], "la pluie est bonne", []), "Garden": (["vegetable garden"], "jardin", ["potager"]), "Cattle / Cows": (["cattle", "cows"], "bétail", ["vaches"]), "Where is the thing?": (["where is it"], "où est la chose ?", []), "To cultivate or to farm": (["to farm", "to cultivate"], "cultiver", []), "To sow or plant seeds": (["to sow", "to plant seeds"], "semer", []), "To harvest": (["harvest"], "récolter", []), "Where is the clinic?": (["where's the clinic", "where is the hospital"], "où est le centre de santé ?", ["où est l'hôpital ?"]), "Take this medicine": (["take the medicine"], "prends ce médicament", []), "Rest now": (["rest", "lie down"], "repose-toi", []), "Are you better?": (["do you feel better"], "ça va mieux ?", []), "How much is this?": (["how much", "what is the price"], "c'est combien ?", ["combien ?"]), "It is too expensive": (["too expensive", "it's too much"], "c'est trop cher", []), "Reduce the price": (["lower the price"], "baisse le prix", ["diminue le prix"]), "Shop / Boutique": (["shop", "store"], "boutique", []), "Matches": (["match"], "allumettes", []), "Where is the road to...?": (["where is the road", "which way to"], "où est la route de", []), "It is near": (["it's near", "it's close"], "c'est proche", []), "Car / Taxi": (["car", "taxi"], "voiture", ["taxi"]), "Bicycle": (["bike"], "vélo", []), "Motorcycle": (["motorbike"], "moto", []), "Please repeat": (["repeat it", "say it again"], "répète", ["répétez s'il vous plaît"]), "Do you speak French?": (["can you speak french"], "tu parles français ?", []), "I can just a little": (["a little", "just a little"], "un peu", []), "See you later": (["until later"], "à plus tard", []), "See you tomorrow": (["until tomorrow"], "à demain", []), } FILES = {"bambara_phrasebook.json": "bam", "pular_phrasebook.json": "ful"} def merge(lst: list[str], new: list[str]) -> list[str]: seen = {_normalize(x) for x in lst} for x in new: if _normalize(x) not in seen: lst.append(x) seen.add(_normalize(x)) return lst def main() -> int: check = "--check" in sys.argv ok = True for fname in FILES: path = DIR / fname data = json.loads(path.read_text(encoding="utf-8")) added, missing = 0, [] for p in data["pairs"]: spec = A.get(p["source"]) if spec is None: missing.append(p["source"]) continue en, fr, frs = spec before = len(p.get("sources", [])) + len(p.get("sources_fr", [])) + bool(p.get("source_fr")) if en: p["sources"] = merge(p.get("sources", []), en) if fr and not p.get("source_fr"): p["source_fr"] = fr if frs: p["sources_fr"] = merge(p.get("sources_fr", []), frs) after = len(p.get("sources", [])) + len(p.get("sources_fr", [])) + bool(p.get("source_fr")) added += after - before # Collision check: one normalised alias -> exactly one target. owner: dict[str, str] = {} for p in data["pairs"]: keys = [p["source"], *p.get("sources", []), p.get("source_fr", ""), *p.get("sources_fr", [])] for k in filter(None, keys): n = _normalize(k) if n in owner and owner[n] != p["target"]: print(f"[{fname}] COLLISION '{k}': {owner[n]!r} vs {p['target']!r}") ok = False owner.setdefault(n, p["target"]) print(f"{fname}: +{added} aliases, {len(owner)} distinct keys, " f"{len(missing)} rows without spec {missing if missing else ''}") if not check and ok: path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") if not ok: print("Aborted: resolve collisions first.") return 1 return 0 if __name__ == "__main__": raise SystemExit(main())