Spaces:
Running
Running
File size: 2,468 Bytes
caf26c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | """
Shared constants used across the gairaigo origin classifier pipeline.
ISO_639_2_NAMES maps the three-letter ISO 639-2 language codes that JMdict
uses in its <lsource> tags to their full English language names. This is used
to replace raw codes like 'fre' or 'ger' with readable labels like 'French'
or 'German' throughout the pipeline — in charts, printed reports, and CSVs.
If a code appears in the data but is not listed here, the code itself is kept
as-is so nothing breaks silently. New codes can simply be added to the dict.
Reference: https://www.loc.gov/standards/iso639-2/php/code_list.php
"""
ISO_639_2_NAMES: dict[str, str] = {
# Most common donor languages in JMdict
"eng": "English",
"fre": "French",
"ger": "German",
"por": "Portuguese",
"dut": "Dutch",
"ita": "Italian",
"spa": "Spanish",
"chi": "Chinese",
"kor": "Korean",
"rus": "Russian",
"ara": "Arabic",
"lat": "Latin",
"grc": "Ancient Greek",
"gre": "Modern Greek",
"san": "Sanskrit",
"ain": "Ainu",
# Less common but present in JMdict
"afr": "Afrikaans",
"alb": "Albanian",
"arm": "Armenian",
"bnt": "Bantu",
"bur": "Burmese",
"cze": "Czech",
"dan": "Danish",
"egy": "Ancient Egyptian",
"epo": "Esperanto",
"fin": "Finnish",
"geo": "Georgian",
"haw": "Hawaiian",
"heb": "Hebrew",
"hin": "Hindi",
"hun": "Hungarian",
"ice": "Icelandic",
"ind": "Indonesian",
"iri": "Irish",
"khm": "Khmer",
"may": "Malay",
"mol": "Moldavian",
"mon": "Mongolian",
"nor": "Norwegian",
"per": "Persian",
"pol": "Polish",
"rum": "Romanian",
"scr": "Croatian",
"slo": "Slovak",
"slv": "Slovenian",
"swa": "Swahili",
"swe": "Swedish",
"tha": "Thai",
"tib": "Tibetan",
"tur": "Turkish",
"ukr": "Ukrainian",
"vie": "Vietnamese",
"wel": "Welsh",
"yid": "Yiddish",
# Catch-all label for consolidated rare classes
"other": "Other",
}
def decode_language(code: str) -> str:
"""
Convert an ISO 639-2 code to a full language name.
Falls back to the code itself if it is not in the mapping table,
so the pipeline never crashes on an unexpected code.
Args:
code : ISO 639-2 three-letter language code (e.g. 'fre').
Returns:
Full language name (e.g. 'French'), or the original code if unknown.
"""
return ISO_639_2_NAMES.get(code, code)
|