Spaces:
Sleeping
Sleeping
| """Add English paraphrase + French aliases to the curated phrasebooks. | |
| Only touches the INPUT side (sources / source_fr / sources_fr). Bambara and | |
| Pular `target` strings are never modified. Idempotent: re-running merges | |
| without duplicating. Aborts if any alias would map to two different targets | |
| in the same phrasebook. | |
| python scripts/add_phrasebook_aliases.py # apply | |
| python scripts/add_phrasebook_aliases.py --check # dry run | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from src.llm.phrasebook import _normalize # noqa: E402 | |
| DIR = Path(__file__).resolve().parent.parent / "configs" / "dialect_anchors" | |
| # canonical English source -> (english aliases, french canonical, french aliases) | |
| A: dict[str, tuple[list[str], str, list[str]]] = { | |
| "Hello / Thank you": (["hello", "hi", "thank you", "thanks"], "bonjour", ["salut", "merci"]), | |
| "Good morning": (["morning", "good morning to you"], "bonjour (le matin)", ["bon matin"]), | |
| "Good afternoon": (["afternoon"], "bon après-midi", []), | |
| "Good evening": (["evening"], "bonsoir", []), | |
| "Welcome": (["you are welcome here", "welcome home"], "bienvenue", ["soyez le bienvenu"]), | |
| "How are you?": (["how are you doing", "how is it going", "are you well"], "comment ça va ?", ["ça va ?", "comment allez-vous ?", "comment vas-tu ?"]), | |
| "Fine, no trouble": (["i am fine", "i'm fine", "fine"], "ça va bien", ["je vais bien"]), | |
| "How was the night?": (["did you sleep well", "how did you sleep"], "as-tu bien dormi ?", ["avez-vous bien dormi ?"]), | |
| "How was the work?": (["how was work", "how is the work"], "comment était le travail ?", []), | |
| "Well done": (["good job", "great work"], "bravo", ["bon travail"]), | |
| "What is your name?": (["what's your name", "your name"], "comment tu t'appelles ?", ["quel est ton nom ?", "comment vous appelez-vous ?"]), | |
| "My name is...": (["my name is", "i am called"], "je m'appelle", ["mon nom est"]), | |
| "Where are you from?": (["where do you come from"], "d'où viens-tu ?", ["d'où venez-vous ?"]), | |
| "I am from...": (["i am from", "i come from"], "je viens de", []), | |
| "What is your work?": (["what do you do", "what is your job"], "quel est ton travail ?", ["que fais-tu ?"]), | |
| "How is the family?": (["how is your family", "how's the family"], "comment va la famille ?", []), | |
| "How is your wife?": (["how's your wife"], "comment va ta femme ?", ["comment va votre femme ?"]), | |
| "How is your husband?": (["how's your husband"], "comment va ton mari ?", ["comment va votre mari ?"]), | |
| "How are the children?": (["how are the kids", "how are your children"], "comment vont les enfants ?", []), | |
| "How is the baby?": (["how's the baby"], "comment va le bébé ?", []), | |
| "They are fine": (["they are well", "they're fine"], "ils vont bien", []), | |
| "My father is well": (["my dad is fine"], "mon père va bien", []), | |
| "My mother is well": (["my mom is fine", "my mum is fine"], "ma mère va bien", []), | |
| "Are you married?": (["are you married"], "es-tu marié ?", ["êtes-vous marié ?"]), | |
| "I am hungry": (["i'm hungry"], "j'ai faim", []), | |
| "I am thirsty": (["i'm thirsty"], "j'ai soif", []), | |
| "I want to eat": (["i would like to eat"], "je veux manger", []), | |
| "Give me water": (["water please", "can i have water"], "donne-moi de l'eau", ["de l'eau s'il vous plaît"]), | |
| "I am full": (["i'm full", "i've eaten enough"], "je suis rassasié", []), | |
| "Bread": ([], "pain", []), | |
| "Rice": ([], "riz", []), | |
| "Meat": ([], "viande", []), | |
| "Tea": ([], "thé", []), | |
| "Sugar": ([], "sucre", []), | |
| "How is the farming?": (["how is the farm", "how are the crops"], "comment va le champ ?", ["comment va la culture ?"]), | |
| "It rained today": (["it rained", "there was rain today"], "il a plu aujourd'hui", []), | |
| "The field": (["field", "farm field"], "le champ", []), | |
| "Maize / Corn": (["maize", "corn"], "maïs", []), | |
| "Cow": (["a cow"], "vache", []), | |
| "Sheep": ([], "mouton", []), | |
| "Goat": ([], "chèvre", []), | |
| "Chicken": (["hen"], "poulet", ["poule"]), | |
| "Where is the hoe?": (["where's the hoe"], "où est la houe ?", ["où est la daba ?"]), | |
| "I am sick": (["i'm sick", "i am ill"], "je suis malade", []), | |
| "My head hurts": (["i have a headache"], "j'ai mal à la tête", []), | |
| "My stomach hurts": (["i have a stomach ache"], "j'ai mal au ventre", []), | |
| "I have fever": (["i have a fever"], "j'ai de la fièvre", []), | |
| "Where is the hospital?": (["where's the hospital"], "où est l'hôpital ?", []), | |
| "Where is the doctor?": (["where's the doctor"], "où est le médecin ?", ["où est le docteur ?"]), | |
| "Take the medicine": (["take your medicine"], "prends le médicament", []), | |
| "Drink this": ([], "bois ceci", ["bois ça"]), | |
| "Lie down": (["lay down"], "couche-toi", []), | |
| "Do you feel better?": (["are you feeling better"], "tu te sens mieux ?", ["ça va mieux ?"]), | |
| "How much?": (["how much is it", "what is the price"], "combien ?", ["c'est combien ?"]), | |
| "It is too much": (["it's too much", "too expensive"], "c'est trop cher", []), | |
| "Reduce it": (["lower the price"], "diminue le prix", ["baisse le prix"]), | |
| "I have no money": (["i don't have money"], "je n'ai pas d'argent", []), | |
| "Here is the money": (["here's the money"], "voici l'argent", []), | |
| "Market": (["the market"], "marché", ["le marché"]), | |
| "Shop": (["store"], "boutique", []), | |
| "Soap": ([], "savon", []), | |
| "Oil": (["cooking oil"], "huile", []), | |
| "Salt": ([], "sel", []), | |
| "Where is the road?": (["which way is the road"], "où est la route ?", []), | |
| "Is it far?": (["is it far away"], "c'est loin ?", []), | |
| "It is close": (["it's close", "it's nearby"], "c'est proche", []), | |
| "Turn right": ([], "tourne à droite", []), | |
| "Turn left": ([], "tourne à gauche", []), | |
| "Stop here": ([], "arrête-toi ici", []), | |
| "Let's go": (["let us go"], "on y va", ["allons-y"]), | |
| "Car": ([], "voiture", []), | |
| "Bus": ([], "bus", []), | |
| "Motorbike": (["motorcycle"], "moto", []), | |
| "I understand": (["i get it"], "je comprends", []), | |
| "I don't understand": (["i do not understand"], "je ne comprends pas", []), | |
| "Repeat it": (["please repeat", "say it again"], "répète", ["répétez s'il vous plaît"]), | |
| "Speak slowly": (["speak slower", "slow down please"], "parle lentement", ["parlez lentement"]), | |
| "Do you speak Bambara?": (["can you speak bambara"], "tu parles bambara ?", []), | |
| "A little": (["a bit"], "un peu", []), | |
| "I don't know": (["i do not know"], "je ne sais pas", []), | |
| "Yes": ([], "oui", []), | |
| "No": ([], "non", []), | |
| "Wait": (["wait a moment"], "attends", ["attendez"]), | |
| "Today": ([], "aujourd'hui", []), | |
| "Tomorrow": ([], "demain", []), | |
| "Yesterday": ([], "hier", []), | |
| "Now": (["right now"], "maintenant", []), | |
| "Later": ([], "plus tard", []), | |
| "Goodbye": (["bye"], "au revoir", []), | |
| "Until later": (["see you later"], "à plus tard", []), | |
| "Until tomorrow": (["see you tomorrow"], "à demain", []), | |
| "Have a good day": ([], "bonne journée", []), | |
| "Have a good night": (["good night"], "bonne nuit", []), | |
| "Go in peace": ([], "va en paix", []), | |
| "God bless you": ([], "que dieu te bénisse", []), | |
| "God willing": (["inshallah"], "si dieu le veut", []), | |
| "Thank God": (["praise god", "alhamdulillah"], "dieu merci", []), | |
| "Peace only": (["only peace"], "la paix seulement", []), | |
| # Pular-only keys | |
| "Peace only (Response)": (["peace only response"], "la paix seulement (réponse)", []), | |
| "Is there any trouble?": (["any trouble", "any problem"], "il y a un problème ?", []), | |
| "The food is sweet": (["the food is good", "the food is delicious"], "la nourriture est bonne", ["c'est bon"]), | |
| "We are working": (["we're working"], "nous travaillons", ["on travaille"]), | |
| "We are working (speaking to the person I'm working with)": ([], "", []), | |
| "We are working (speaking to another person not working with us)": ([], "", []), | |
| "No trouble": (["no problem"], "pas de problème", []), | |
| "How is the heat/weather?": (["how is the heat", "how is the weather"], "comment est la chaleur ?", ["quel temps fait-il ?"]), | |
| "Where are you coming from?": (["where are you coming from"], "d'où arrives-tu ?", []), | |
| "I am coming from...": (["i am coming from"], "j'arrive de", []), | |
| "I am from": (["i come from"], "je viens de", []), | |
| "I am a farmer": (["i'm a farmer"], "je suis cultivateur", ["je suis agriculteur"]), | |
| "How is the woman?": ([], "comment va la femme ?", []), | |
| "How is the man?": ([], "comment va l'homme ?", []), | |
| "Everyone is fine": (["everybody is fine", "all are well"], "tout le monde va bien", []), | |
| "How many children?": (["how many kids"], "combien d'enfants ?", []), | |
| "The food is good": (["the food is delicious"], "la nourriture est bonne", ["c'est bon"]), | |
| "Milk": ([], "lait", []), | |
| "Sour Cream": (["sour milk"], "lait caillé", []), | |
| "Hot water": ([], "eau chaude", []), | |
| "Cold water": ([], "eau froide", []), | |
| "Coffee": ([], "café", []), | |
| "The rain is good": (["good rain"], "la pluie est bonne", []), | |
| "Garden": (["vegetable garden"], "jardin", ["potager"]), | |
| "Cattle / Cows": (["cattle", "cows"], "bétail", ["vaches"]), | |
| "Where is the thing?": (["where is it"], "où est la chose ?", []), | |
| "To cultivate or to farm": (["to farm", "to cultivate"], "cultiver", []), | |
| "To sow or plant seeds": (["to sow", "to plant seeds"], "semer", []), | |
| "To harvest": (["harvest"], "récolter", []), | |
| "Where is the clinic?": (["where's the clinic", "where is the hospital"], "où est le centre de santé ?", ["où est l'hôpital ?"]), | |
| "Take this medicine": (["take the medicine"], "prends ce médicament", []), | |
| "Rest now": (["rest", "lie down"], "repose-toi", []), | |
| "Are you better?": (["do you feel better"], "ça va mieux ?", []), | |
| "How much is this?": (["how much", "what is the price"], "c'est combien ?", ["combien ?"]), | |
| "It is too expensive": (["too expensive", "it's too much"], "c'est trop cher", []), | |
| "Reduce the price": (["lower the price"], "baisse le prix", ["diminue le prix"]), | |
| "Shop / Boutique": (["shop", "store"], "boutique", []), | |
| "Matches": (["match"], "allumettes", []), | |
| "Where is the road to...?": (["where is the road", "which way to"], "où est la route de", []), | |
| "It is near": (["it's near", "it's close"], "c'est proche", []), | |
| "Car / Taxi": (["car", "taxi"], "voiture", ["taxi"]), | |
| "Bicycle": (["bike"], "vélo", []), | |
| "Motorcycle": (["motorbike"], "moto", []), | |
| "Please repeat": (["repeat it", "say it again"], "répète", ["répétez s'il vous plaît"]), | |
| "Do you speak French?": (["can you speak french"], "tu parles français ?", []), | |
| "I can just a little": (["a little", "just a little"], "un peu", []), | |
| "See you later": (["until later"], "à plus tard", []), | |
| "See you tomorrow": (["until tomorrow"], "à demain", []), | |
| } | |
| FILES = {"bambara_phrasebook.json": "bam", "pular_phrasebook.json": "ful"} | |
| def merge(lst: list[str], new: list[str]) -> list[str]: | |
| seen = {_normalize(x) for x in lst} | |
| for x in new: | |
| if _normalize(x) not in seen: | |
| lst.append(x) | |
| seen.add(_normalize(x)) | |
| return lst | |
| def main() -> int: | |
| check = "--check" in sys.argv | |
| ok = True | |
| for fname in FILES: | |
| path = DIR / fname | |
| data = json.loads(path.read_text(encoding="utf-8")) | |
| added, missing = 0, [] | |
| for p in data["pairs"]: | |
| spec = A.get(p["source"]) | |
| if spec is None: | |
| missing.append(p["source"]) | |
| continue | |
| en, fr, frs = spec | |
| before = len(p.get("sources", [])) + len(p.get("sources_fr", [])) + bool(p.get("source_fr")) | |
| if en: | |
| p["sources"] = merge(p.get("sources", []), en) | |
| if fr and not p.get("source_fr"): | |
| p["source_fr"] = fr | |
| if frs: | |
| p["sources_fr"] = merge(p.get("sources_fr", []), frs) | |
| after = len(p.get("sources", [])) + len(p.get("sources_fr", [])) + bool(p.get("source_fr")) | |
| added += after - before | |
| # Collision check: one normalised alias -> exactly one target. | |
| owner: dict[str, str] = {} | |
| for p in data["pairs"]: | |
| keys = [p["source"], *p.get("sources", []), p.get("source_fr", ""), *p.get("sources_fr", [])] | |
| for k in filter(None, keys): | |
| n = _normalize(k) | |
| if n in owner and owner[n] != p["target"]: | |
| print(f"[{fname}] COLLISION '{k}': {owner[n]!r} vs {p['target']!r}") | |
| ok = False | |
| owner.setdefault(n, p["target"]) | |
| print(f"{fname}: +{added} aliases, {len(owner)} distinct keys, " | |
| f"{len(missing)} rows without spec {missing if missing else ''}") | |
| if not check and ok: | |
| path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| if not ok: | |
| print("Aborted: resolve collisions first.") | |
| return 1 | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |